ensure valid ulimit names

Add customizable ulimit settings for all Docker containers

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -2,13 +2,11 @@ body:
   - type: markdown
     attributes:
       value: |
-        ⚠️ This category is solely for conversations related to Security Onion 2.4 ⚠️
-
         If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
   - type: dropdown
     attributes:
       label: Version
-      description: Which version of Security Onion 2.4.x are you asking about?
+      description: Which version of Security Onion are you asking about?
       options:
         -
         - 2.4.10
@@ -33,6 +31,9 @@ body:
         - 2.4.180
         - 2.4.190
         - 2.4.200
+        - 2.4.201
+        - 2.4.210
+        - 2.4.211
         - Other (please provide detail below)
     validations:
       required: true
@@ -94,7 +95,7 @@ body:
     attributes:
       label: Hardware Specs
       description: >
-        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://docs.securityonion.net/en/2.4/hardware.html?
+        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://securityonion.net/docs/hardware?
       options:
         -
         - Meets minimum requirements

.github/DISCUSSION_TEMPLATE/3-0.yml

@@ -0,0 +1,177 @@
+body:
+  - type: markdown
+    attributes:
+      value: |
+        If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
+  - type: dropdown
+    attributes:
+      label: Version
+      description: Which version of Security Onion are you asking about?
+      options:
+        -
+        - 3.0.0
+        - Other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Installation Method
+      description: How did you install Security Onion?
+      options:
+        -
+        - Security Onion ISO image
+        - Cloud image (Amazon, Azure, Google)
+        - Network installation on Oracle 9 (unsupported)
+        - Other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Description
+      description: >
+        Is this discussion about installation, configuration, upgrading, or other?
+      options:
+        -
+        - installation
+        - configuration
+        - upgrading
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Installation Type
+      description: >
+        When you installed, did you choose Import, Eval, Standalone, Distributed, or something else?
+      options:
+        -
+        - Import
+        - Eval
+        - Standalone
+        - Distributed
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Location
+      description: >
+        Is this deployment in the cloud, on-prem with Internet access, or airgap?
+      options:
+        -
+        - cloud
+        - on-prem with Internet access
+        - airgap
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Hardware Specs
+      description: >
+        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://securityonion.net/docs/hardware?
+      options:
+        -
+        - Meets minimum requirements
+        - Exceeds minimum requirements
+        - Does not meet minimum requirements
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: CPU
+      description: How many CPU cores do you have?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: RAM
+      description: How much RAM do you have?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Storage for /
+      description: How much storage do you have for the / partition?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Storage for /nsm
+      description: How much storage do you have for the /nsm partition?
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Network Traffic Collection
+      description: >
+        Are you collecting network traffic from a tap or span port?
+      options:
+        -
+        - tap
+        - span port
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Network Traffic Speeds
+      description: >
+        How much network traffic are you monitoring?
+      options:
+        -
+        - Less than 1Gbps
+        - 1Gbps to 10Gbps
+        - more than 10Gbps
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Status
+      description: >
+        Does SOC Grid show all services on all nodes as running OK?
+      options:
+        -
+        - Yes, all services on all nodes are running OK
+        - No, one or more services are failed (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Salt Status
+      description: >
+        Do you get any failures when you run "sudo salt-call state.highstate"?
+      options:
+        -
+        - Yes, there are salt failures (please provide detail below)
+        - No, there are no failures
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Logs
+      description: >
+        Are there any additional clues in /opt/so/log/?
+      options:
+        -
+        - Yes, there are additional clues in /opt/so/log/ (please provide detail below)
+        - No, there are no additional clues
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Detail
+      description: Please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and then provide detailed information to help us help you.
+      placeholder: |-
+        STOP! Before typing, please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 in their entirety!
+
+        If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
+    validations:
+      required: true
+  - type: checkboxes
+    attributes:
+      label: Guidelines
+      options:
+        - label: I have read the discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and assert that I have followed the guidelines.
+          required: true

.github/workflows/pythontest.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.13"]
+        python-version: ["3.14"]
         python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"]
 
     steps:

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.190-20251024 ISO image released on 2025/10/24
+### 2.4.210-20260302 ISO image released on 2026/03/02
 
 
 ### Download and Verify
 
-2.4.190-20251024 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.190-20251024.iso
+2.4.210-20260302 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.210-20260302.iso
  
-MD5: 25358481FB876226499C011FC0710358  
-SHA1: 0B26173C0CE136F2CA40A15046D1DFB78BCA1165  
-SHA256: 4FD9F62EDA672408828B3C0C446FE5EA9FF3C4EE8488A7AB1101544A3C487872  
+MD5: 575F316981891EBED2EE4E1F42A1F016  
+SHA1: 600945E8823221CBC5F1C056084A71355308227E  
+SHA256: A6AA6471125F07FA6E2796430E94BEAFDEF728E833E9728FDFA7106351EBC47E  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.190-20251024.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.210-20260302.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.190-20251024.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.210-20260302.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.190-20251024.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.210-20260302.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.190-20251024.iso.sig securityonion-2.4.190-20251024.iso
+gpg --verify securityonion-2.4.210-20260302.iso.sig securityonion-2.4.210-20260302.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Thu 23 Oct 2025 07:21:46 AM EDT using RSA key ID FE507013
+gpg: Signature made Mon 02 Mar 2026 11:55:24 AM EST using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.
@@ -50,4 +50,4 @@ Primary key fingerprint: C804 A93D 36BE 0C73 3EA1  9644 7C10 60B7 FE50 7013
 If it fails to verify, try downloading again. If it still fails to verify, try downloading from another computer or another network.
 
 Once you've verified the ISO image, you're ready to proceed to our Installation guide:  
-https://docs.securityonion.net/en/2.4/installation.html
+https://securityonion.net/docs/installation

README.md

@@ -1,50 +1,58 @@
-## Security Onion 2.4
+<p align="center">
+  <img src="https://securityonionsolutions.com/logo/logo-so-onion-dark.svg" width="400" alt="Security Onion Logo">
+</p>
 
-Security Onion 2.4 is here!
+# Security Onion
 
-## Screenshots
+Security Onion is a free and open Linux distribution for threat hunting, enterprise security monitoring, and log management. It includes a comprehensive suite of tools designed to work together to provide visibility into your network and host activity.
 
-Alerts
-![Alerts](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/50_alerts.png)
+## ✨ Features
 
-Dashboards
-![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_dashboards.png)
+Security Onion includes everything you need to monitor your network and host systems:
 
-Hunt
-![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/56_hunt.png)
+*   **Security Onion Console (SOC)**: A unified web interface for analyzing security events and managing your grid.
+*   **Elastic Stack**: Powerful search backed by Elasticsearch.
+*   **Intrusion Detection**: Network-based IDS with Suricata and host-based monitoring with Elastic Fleet.
+*   **Network Metadata**: Detailed network metadata generated by Zeek or Suricata.
+*   **Full Packet Capture**: Retain and analyze raw network traffic with Suricata PCAP.
 
-Detections
-![Detections](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_detections.png)
+## ⭐ Security Onion Pro
 
-PCAP
-![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/62_pcap.png)
+For organizations and enterprises requiring advanced capabilities, **Security Onion Pro** offers additional features designed for scale and efficiency:
 
-Grid
-![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/75_grid.png)
+*   **Onion AI**: Leverage powerful AI-driven insights to accelerate your analysis and investigations.
+*   **Enterprise Features**: Enhanced tools and integrations tailored for enterprise-grade security operations.
 
-Config
-![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/87_config.png)
+For more information, visit the [Security Onion Pro](https://securityonionsolutions.com/pro) page.
 
-### Release Notes
+## ☁️ Cloud Deployment
 
-https://docs.securityonion.net/en/2.4/release-notes.html
+Security Onion is available and ready to deploy in the **AWS**, **Azure**, and **Google Cloud (GCP)** marketplaces.
 
-### Requirements
+## 🚀 Getting Started
 
-https://docs.securityonion.net/en/2.4/hardware.html
+| Goal | Resource |
+| :--- | :--- |
+| **Download** | [Security Onion ISO](https://securityonion.net/docs/download) |
+| **Requirements** | [Hardware Guide](https://securityonion.net/docs/hardware) |
+| **Install** | [Installation Instructions](https://securityonion.net/docs/installation) |
+| **What's New** | [Release Notes](https://securityonion.net/docs/release-notes) |
 
-### Download
+## 📖 Documentation & Support
 
-https://docs.securityonion.net/en/2.4/download.html
+For more detailed information, please visit our [Documentation](https://docs.securityonion.net).
 
-### Installation
+*   **FAQ**: [Frequently Asked Questions](https://securityonion.net/docs/faq)
+*   **Community**: [Discussions & Support](https://securityonion.net/docs/community-support)
+*   **Training**: [Official Training](https://securityonion.net/training)
 
-https://docs.securityonion.net/en/2.4/installation.html
+## 🤝 Contributing
 
-### FAQ
+We welcome contributions! Please see our [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on how to get involved.
 
-https://docs.securityonion.net/en/2.4/faq.html
+## 🛡️ License
 
-### Feedback
+Security Onion is licensed under the terms of the license found in the [LICENSE](LICENSE) file.
 
-https://docs.securityonion.net/en/2.4/community-support.html
+---
+*Built with 🧅 by Security Onion Solutions.*

SECURITY.md

@@ -4,6 +4,7 @@
 
 | Version | Supported          |
 | ------- | ------------------ |
+| 3.x     | :white_check_mark: |
 | 2.4.x   | :white_check_mark: |
 | 2.3.x   | :x:                |
 | 16.04.x | :x:                |

VERSION

@@ -1 +1 @@
-2.4.200
+3.0.0

pillar/ca/init.sls

@@ -0,0 +1,2 @@
+ca:
+  server:

pillar/top.sls

@@ -1,5 +1,6 @@
 base:
   '*':
+    - ca
     - global.soc_global
     - global.adv_global
     - docker.soc_docker
@@ -43,8 +44,6 @@ base:
     - secrets
     - manager.soc_manager
     - manager.adv_manager
-    - idstools.soc_idstools
-    - idstools.adv_idstools
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
@@ -88,8 +87,6 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
-    - pcap.soc_pcap
-    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - minions.{{ grains.id }}
@@ -117,8 +114,6 @@ base:
     - elastalert.adv_elastalert
     - manager.soc_manager
     - manager.adv_manager
-    - idstools.soc_idstools
-    - idstools.adv_idstools
     - soc.soc_soc
     - soc.adv_soc
     - kibana.soc_kibana
@@ -137,8 +132,6 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
-    - pcap.soc_pcap
-    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - minions.{{ grains.id }}
@@ -158,8 +151,6 @@ base:
     {% endif %}
     - secrets
     - healthcheck.standalone
-    - idstools.soc_idstools
-    - idstools.adv_idstools
     - kratos.soc_kratos
     - kratos.adv_kratos
     - hydra.soc_hydra
@@ -190,8 +181,6 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
-    - pcap.soc_pcap
-    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - minions.{{ grains.id }}
@@ -214,8 +203,6 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
-    - pcap.soc_pcap
-    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - strelka.soc_strelka
@@ -302,8 +289,6 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
-    - pcap.soc_pcap
-    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - strelka.soc_strelka

salt/_modules/needs_restarting.py

@@ -1,24 +1,14 @@
-from os import path
 import subprocess
 
 def check():
 
-  osfam = __grains__['os_family']
   retval = 'False'
 
-  if osfam == 'Debian':
-    if path.exists('/var/run/reboot-required'):
-      retval = 'True'
+  cmd = 'needs-restarting -r > /dev/null 2>&1'
 
-  elif osfam == 'RedHat':
-    cmd = 'needs-restarting -r > /dev/null 2>&1'
-    
-    try:
-      needs_restarting = subprocess.check_call(cmd, shell=True)
-    except subprocess.CalledProcessError:
-      retval = 'True'
-
-  else:
-    retval = 'Unsupported OS: %s' % os
+  try:
+    needs_restarting = subprocess.check_call(cmd, shell=True)
+  except subprocess.CalledProcessError:
+    retval = 'True'
 
   return retval

salt/allowed_states.map.jinja

@@ -15,11 +15,7 @@
     'salt.minion-check',
     'sensoroni',
     'salt.lasthighstate',
-    'salt.minion'
-] %}
-
-{% set ssl_states = [
-    'ssl',
+    'salt.minion',
     'telegraf',
     'firewall',
     'schedule',
@@ -28,7 +24,7 @@
 
 {% set manager_states = [
     'salt.master',
-    'ca',
+    'ca.server',
     'registry',
     'manager',
     'nginx',
@@ -38,13 +34,10 @@
     'hydra',
     'elasticfleet',
     'elastic-fleet-package-registry',
-    'idstools',
-    'suricata.manager',
     'utility'
 ] %}
 
 {% set sensor_states = [
-    'pcap',
     'suricata',
     'healthcheck',
     'tcpreplay',
@@ -77,28 +70,24 @@
     {# Map role-specific states #}
     {% set role_states = {
         'so-eval': (
-            ssl_states +
             manager_states +
             sensor_states +
-            elastic_stack_states | reject('equalto', 'logstash') | list
+            elastic_stack_states | reject('equalto', 'logstash') | list +
+            ['logstash.ssl']
         ),
         'so-heavynode': (
-            ssl_states +
             sensor_states +
             ['elasticagent', 'elasticsearch', 'logstash', 'redis', 'nginx']
         ),
         'so-idh': (
-            ssl_states +
             ['idh']
         ),
         'so-import': (
-            ssl_states +
             manager_states +
             sensor_states | reject('equalto', 'strelka') | reject('equalto', 'healthcheck') | list +
-            ['elasticsearch', 'elasticsearch.auth', 'kibana', 'kibana.secrets', 'strelka.manager']
+            ['elasticsearch', 'elasticsearch.auth', 'kibana', 'kibana.secrets', 'logstash.ssl', 'strelka.manager']
         ),
         'so-manager': (
-            ssl_states +
             manager_states +
             ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users', 'strelka.manager'] +
             stig_states +
@@ -106,7 +95,6 @@
             elastic_stack_states
         ),
         'so-managerhype': (
-            ssl_states +
             manager_states +
             ['salt.cloud', 'strelka.manager', 'hypervisor', 'libvirt'] +
             stig_states +
@@ -114,7 +102,6 @@
             elastic_stack_states
         ),
         'so-managersearch': (
-            ssl_states +
             manager_states +
             ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users', 'strelka.manager'] +
             stig_states +
@@ -122,12 +109,10 @@
             elastic_stack_states
         ),
         'so-searchnode': (
-            ssl_states +
             ['kafka.ca', 'kafka.ssl', 'elasticsearch', 'logstash', 'nginx'] +
             stig_states
         ),
         'so-standalone': (
-            ssl_states +
             manager_states +
             ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users'] +
             sensor_states +
@@ -136,29 +121,24 @@
             elastic_stack_states
         ),
         'so-sensor': (
-            ssl_states +
             sensor_states +
             ['nginx'] +
             stig_states
         ),
         'so-fleet': (
-            ssl_states +
             stig_states +
             ['logstash', 'nginx', 'healthcheck', 'elasticfleet']
         ),
         'so-receiver': (
-            ssl_states +
             kafka_states +
             stig_states +
             ['logstash', 'redis']
         ),
         'so-hypervisor': (
-            ssl_states +
             stig_states +
             ['hypervisor', 'libvirt']
         ),
         'so-desktop': (
-            ['ssl', 'docker_clean', 'telegraf'] +
             stig_states
         )
     } %}

salt/bpf/macros.jinja

@@ -1,10 +1,12 @@
 {% macro remove_comments(bpfmerged, app) %}
 
 {# remove comments from the bpf #}
+{% set app_list = [] %}
 {% for bpf in bpfmerged[app] %}
-{%   if bpf.strip().startswith('#') %}
-{%     do bpfmerged[app].pop(loop.index0) %}
+{%   if not bpf.strip().startswith('#') %}
+{%     do app_list.append(bpf) %}
 {%   endif %}
 {% endfor %}
+{% do bpfmerged.update({app: app_list}) %}
 
 {% endmacro %}

salt/bpf/pcap.map.jinja

@@ -1,21 +1,15 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% set PCAP_BPF_STATUS = 0  %}
-{% set STENO_BPF_COMPILED = "" %}
 
-{% if GLOBALS.pcap_engine == "TRANSITION" %}
-{%   set PCAPBPF = ["ip and host 255.255.255.1 and port 1"] %}
-{% else %}
 {%   import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {%   set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
 {%   import 'bpf/macros.jinja' as MACROS %}
 {{   MACROS.remove_comments(BPFMERGED, 'pcap') }}
 {%   set PCAPBPF = BPFMERGED.pcap %}
-{% endif %}
 
 {% if PCAPBPF %}
-   {% set PCAP_BPF_CALC = salt['cmd.run_all']('/usr/sbin/so-bpf-compile ' ~ GLOBALS.sensor.interface ~ ' ' ~ PCAPBPF|join(" "), cwd='/root') %}
+   {% set PCAP_BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + PCAPBPF|join(" "),cwd='/root') %}
    {% if PCAP_BPF_CALC['retcode'] == 0 %}
       {% set PCAP_BPF_STATUS = 1 %}
-      {% set STENO_BPF_COMPILED =  ",\\\"--filter=" + PCAP_BPF_CALC['stdout'] + "\\\""  %}
    {% endif %}
 {% endif %}

salt/bpf/suricata.map.jinja

@@ -9,7 +9,7 @@
 {% set SURICATABPF = BPFMERGED.suricata %}
 
 {% if SURICATABPF %}
-   {% set SURICATA_BPF_CALC = salt['cmd.run_all']('/usr/sbin/so-bpf-compile ' ~ GLOBALS.sensor.interface ~ ' ' ~ SURICATABPF|join(" "), cwd='/root') %}
+   {% set SURICATA_BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + SURICATABPF|join(" "),cwd='/root') %}
    {% if SURICATA_BPF_CALC['retcode'] == 0 %}
       {% set SURICATA_BPF_STATUS = 1 %}
    {% endif %}

salt/bpf/zeek.map.jinja

@@ -9,7 +9,7 @@
 {% set ZEEKBPF = BPFMERGED.zeek %}
 
 {% if ZEEKBPF %}
-   {% set ZEEK_BPF_CALC = salt['cmd.run_all']('/usr/sbin/so-bpf-compile ' ~ GLOBALS.sensor.interface ~ ' ' ~ ZEEKBPF|join(" "), cwd='/root') %}
+   {% set ZEEK_BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + ZEEKBPF|join(" "),cwd='/root') %}
    {% if ZEEK_BPF_CALC['retcode'] == 0 %}
       {% set ZEEK_BPF_STATUS = 1 %}
    {% endif %}

salt/ca/dirs.sls

@@ -1,4 +0,0 @@
-pki_issued_certs:
-  file.directory:
-    - name: /etc/pki/issued_certs
-    - makedirs: True

salt/ca/init.sls

@@ -3,70 +3,10 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
-
 include:
-  - ca.dirs
-
-/etc/salt/minion.d/signing_policies.conf:
-  file.managed:
-    - source: salt://ca/files/signing_policies.conf
-
-pki_private_key:
-  x509.private_key_managed:
-    - name: /etc/pki/ca.key
-    - keysize: 4096
-    - passphrase:
-    - backup: True
-    {% if salt['file.file_exists']('/etc/pki/ca.key') -%}
-    - prereq:
-      - x509: /etc/pki/ca.crt
-    {%- endif %}
-
-pki_public_ca_crt:
-  x509.certificate_managed:
-    - name: /etc/pki/ca.crt
-    - signing_private_key: /etc/pki/ca.key
-    - CN: {{ GLOBALS.manager }}
-    - C: US
-    - ST: Utah
-    - L: Salt Lake City
-    - basicConstraints: "critical CA:true"
-    - keyUsage: "critical cRLSign, keyCertSign"
-    - extendedkeyUsage: "serverAuth, clientAuth"
-    - subjectKeyIdentifier: hash
-    - authorityKeyIdentifier: keyid:always, issuer
-    - days_valid: 3650
-    - days_remaining: 7
-    - backup: True
-    - replace: False
-    - require:
-      - sls: ca.dirs
-    - timeout: 30
-    - retry:
-        attempts: 5
-        interval: 30
-
-mine_update_ca_crt:
-  module.run:
-    - mine.update: []
-    - onchanges:
-      - x509: pki_public_ca_crt
-
-cakeyperms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/ca.key
-    - mode: 640
-    - group: 939
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
+{% if GLOBALS.is_manager %}
+  - ca.server
 {% endif %}
+  - ca.trustca

salt/ca/map.jinja

@@ -0,0 +1,3 @@
+{% set CA = {
+    'server': pillar.ca.server
+}%}

salt/ca/remove.sls

@@ -1,7 +1,35 @@
-pki_private_key:
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% set setup_running = salt['cmd.retcode']('pgrep -x so-setup') == 0 %}
+
+{% if setup_running%}
+
+include:
+  - ssl.remove
+
+remove_pki_private_key:
   file.absent:
     - name: /etc/pki/ca.key
 
-pki_public_ca_crt:
+remove_pki_public_ca_crt:
   file.absent:
     - name: /etc/pki/ca.crt
+
+remove_trusttheca:
+  file.absent:
+    - name: /etc/pki/tls/certs/intca.crt
+
+remove_pki_public_ca_crt_symlink:
+  file.absent:
+    - name: /opt/so/saltstack/local/salt/ca/files/ca.crt
+
+{% else %}
+
+so-setup_not_running:
+  test.show_notification:
+    - text: "This state is reserved for usage during so-setup."
+
+{% endif %}

salt/ca/server.sls

@@ -0,0 +1,63 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+
+pki_private_key:
+  x509.private_key_managed:
+    - name: /etc/pki/ca.key
+    - keysize: 4096
+    - passphrase:
+    - backup: True
+    {% if salt['file.file_exists']('/etc/pki/ca.key') -%}
+    - prereq:
+      - x509: /etc/pki/ca.crt
+    {%- endif %}
+
+pki_public_ca_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/ca.crt
+    - signing_private_key: /etc/pki/ca.key
+    - CN: {{ GLOBALS.manager }}
+    - C: US
+    - ST: Utah
+    - L: Salt Lake City
+    - basicConstraints: "critical CA:true"
+    - keyUsage: "critical cRLSign, keyCertSign"
+    - extendedkeyUsage: "serverAuth, clientAuth"
+    - subjectKeyIdentifier: hash
+    - authorityKeyIdentifier: keyid:always, issuer
+    - days_valid: 3650
+    - days_remaining: 7
+    - backup: True
+    - replace: False
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+
+pki_public_ca_crt_symlink:
+  file.symlink:
+    - name: /opt/so/saltstack/local/salt/ca/files/ca.crt
+    - target: /etc/pki/ca.crt
+    - require:
+      - x509: pki_public_ca_crt
+
+cakeyperms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/ca.key
+    - mode: 640
+    - group: 939
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/ca/signing_policy.sls

@@ -1,12 +1,15 @@
-#!/bin/bash
-
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+# when the salt-minion signs the cert, a copy is stored here
+issued_certs_copypath:
+  file.directory:
+    - name: /etc/pki/issued_certs
+    - makedirs: True
 
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-start idstools $1
+signing_policy:
+  file.managed:
+    - name: /etc/salt/minion.d/signing_policies.conf
+    - source: salt://ca/files/signing_policies.conf

salt/ca/trustca.sls

@@ -3,11 +3,16 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{% from 'idstools/map.jinja' import IDSTOOLSMERGED %}
-
 include:
-{% if IDSTOOLSMERGED.enabled %}
-  - idstools.enabled
-{% else %}
-  - idstools.disabled
-{% endif %}
+  - docker
+
+# Trust the CA
+trusttheca:
+  file.managed:
+    - name: /etc/pki/tls/certs/intca.crt
+    - source: salt://ca/files/ca.crt
+    - watch_in:
+      - service: docker_running
+    - show_changes: False
+    - makedirs: True
+

salt/common/files/daemon.json

@@ -1,12 +0,0 @@
-{
-  "registry-mirrors": [
-    "https://:5000"
-  ],
-  "bip": "172.17.0.1/24",
-  "default-address-pools": [
-    {
-      "base": "172.17.0.0/24",
-      "size": 24
-    }
-  ]
-}

salt/common/init.sls

@@ -20,11 +20,6 @@ kernel.printk:
   sysctl.present:
     - value: "3 4 1 3"
 
-# Remove variables.txt from /tmp - This is temp
-rmvariablesfile:
-  file.absent:
-    - name: /tmp/variables.txt
-
 # Add socore Group
 socoregroup:
   group.present:
@@ -149,35 +144,13 @@ common_sbin_jinja:
       - so-import-pcap
 {% endif %}
 
-{% if GLOBALS.role == 'so-heavynode' %}
-remove_so-pcap-import_heavynode:
-  file.absent:
-    - name: /usr/sbin/so-pcap-import
-
-remove_so-import-pcap_heavynode:
-  file.absent:
-    - name: /usr/sbin/so-import-pcap
-{% endif %}
-
-{% if not GLOBALS.is_manager%}
-# prior to 2.4.50 these scripts were in common/tools/sbin on the manager because of soup and distributed to non managers
-# these two states remove the scripts from non manager nodes
-remove_soup:
-  file.absent:
-    - name: /usr/sbin/soup
-
-remove_so-firewall:
-  file.absent:
-    - name: /usr/sbin/so-firewall
-{% endif %}
-
 so-status_script:
   file.managed:
     - name: /usr/sbin/so-status
     - source: salt://common/tools/sbin/so-status
     - mode: 755
 
-{% if GLOBALS.role in GLOBALS.sensor_roles %}
+{% if GLOBALS.is_sensor %}
 # Add sensor cleanup
 so-sensor-clean:
   cron.present:

salt/common/packages.sls

@@ -1,52 +1,5 @@
 # we cannot import GLOBALS from vars/globals.map.jinja in this state since it is called in setup.virt.init
 # since it is early in setup of a new VM, the pillars imported in GLOBALS are not yet defined
-{% if grains.os_family == 'Debian' %}
-commonpkgs:
-  pkg.installed:
-    - skip_suggestions: True
-    - pkgs:
-      - apache2-utils
-      - wget
-      - ntpdate
-      - jq
-      - curl
-      - ca-certificates
-      - software-properties-common
-      - apt-transport-https
-      - openssl
-      - netcat-openbsd
-      - sqlite3
-      - libssl-dev
-      - procps
-      - python3-dateutil
-      - python3-docker
-      - python3-packaging
-      - python3-lxml
-      - git
-      - rsync
-      - vim
-      - tar
-      - unzip
-      - bc
-      {% if grains.oscodename != 'focal' %}
-      - python3-rich
-      {% endif %}
-
-{%     if grains.oscodename == 'focal' %}
-# since Ubuntu requires and internet connection we can use pip to install modules
-python3-pip:
-  pkg.installed
-
-python-rich:
-  pip.installed:
-    - name: rich
-    - target: /usr/local/lib/python3.8/dist-packages/
-    - require:
-      - pkg: python3-pip
-{%     endif %}
-{% endif %}
-
-{% if grains.os_family == 'RedHat' %}
 
 remove_mariadb:
   pkg.removed:
@@ -84,5 +37,3 @@ commonpkgs:
       - unzip
       - wget
       - yum-utils
-
-{% endif %}

salt/common/soup_scripts.sls

@@ -3,8 +3,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{% if '2.4' in salt['cp.get_file_str']('/etc/soversion') %}
-
 {%   import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %}
 {%   if SOC_GLOBAL.global.airgap %}
 {%     set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %}
@@ -13,14 +11,6 @@
 {%   endif %}
 {%   set SOVERSION = salt['file.read']('/etc/soversion').strip() %}
 
-remove_common_soup:
-  file.absent:
-    - name: /opt/so/saltstack/default/salt/common/tools/sbin/soup
-
-remove_common_so-firewall:
-  file.absent:
-    - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall
-
 # This section is used to put the scripts in place in the Salt file system
 # in case a state run tries to overwrite what we do in the next section.
 copy_so-common_common_tools_sbin:
@@ -120,23 +110,3 @@ copy_bootstrap-salt_sbin:
     - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
     - force: True
     - preserve: True
-
-{# this is added in 2.4.120 to remove salt repo files pointing to saltproject.io to accomodate the move to broadcom and new bootstrap-salt script #}
-{%   if salt['pkg.version_cmp'](SOVERSION, '2.4.120') == -1 %}
-{%     set saltrepofile = '/etc/yum.repos.d/salt.repo' %}
-{%     if grains.os_family == 'Debian' %}
-{%       set saltrepofile = '/etc/apt/sources.list.d/salt.list' %}
-{%     endif %}
-remove_saltproject_io_repo_manager:
-  file.absent:
-    - name: {{ saltrepofile }}
-{%   endif %}
-
-{% else %}
-fix_23_soup_sbin:
-  cmd.run:
-    - name: curl -s -f -o /usr/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
-fix_23_soup_salt:
-  cmd.run:
-    - name: curl -s -f -o /opt/so/saltstack/defalt/salt/common/tools/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
-{% endif %}

salt/common/tools/sbin/so-bpf-compile

@@ -16,7 +16,7 @@
 
 if [ "$#" -lt 2 ]; then
   cat 1>&2 <<EOF
-$0 compiles a BPF expression to be passed to stenotype to apply a socket filter.
+$0 compiles a BPF expression to be passed to PCAP to apply a socket filter.
 Its first argument is the interface (link type is required) and all other arguments
 are passed to TCPDump.
 

salt/common/tools/sbin/so-checkin

@@ -10,7 +10,7 @@
 cat << EOF
 
 so-checkin will run a full salt highstate to apply all salt states. If a highstate is already running, this request will be queued and so it may pause for a few minutes before you see any more output. For more information about so-checkin and salt, please see:
-https://docs.securityonion.net/en/2.4/salt.html
+https://securityonion.net/docs/salt
 
 EOF
 

salt/common/tools/sbin/so-common

@@ -10,7 +10,7 @@
 # and since this same logic is required during installation, it's included in this file.
 
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
-DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
+DOC_BASE_URL="https://securityonion.net/docs"
 
 if [ -z $NOROOT ]; then
 	# Check for prerequisites
@@ -333,8 +333,8 @@ get_elastic_agent_vars() {
 
 	if [ -f "$defaultsfile" ]; then
 		ELASTIC_AGENT_TARBALL_VERSION=$(egrep " +version: " $defaultsfile | awk -F: '{print $2}' | tr -d '[:space:]')
-		ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
-		ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+		ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/3/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+		ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/3/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
 		ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
 		ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
 		ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
@@ -349,21 +349,16 @@ get_random_value() {
 }
 
 gpg_rpm_import() {
-	if [[ $is_oracle ]]; then
-	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
-	        local RPMKEYSLOC="../salt/repo/client/files/$OS/keys"
-	    else
-	        local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys"
-	    fi
-		RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
-		for RPMKEY in "${RPMKEYS[@]}"; do
-    	        rpm --import $RPMKEYSLOC/$RPMKEY
-	        echo "Imported $RPMKEY"
-	    done
-	elif [[ $is_rpm ]]; then
-		echo "Importing the security onion GPG key"
-		rpm --import ../salt/repo/client/files/oracle/keys/securityonion.pub
+	if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
+		local RPMKEYSLOC="../salt/repo/client/files/$OS/keys"
+	else
+		local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys"
 	fi
+	RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
+	for RPMKEY in "${RPMKEYS[@]}"; do
+		rpm --import $RPMKEYSLOC/$RPMKEY
+		echo "Imported $RPMKEY"
+	done
 }
 
 header() {
@@ -404,6 +399,25 @@ is_single_node_grid() {
 	grep "role: so-" /etc/salt/grains | grep -E "eval|standalone|import" &> /dev/null
 }
 
+initialize_elasticsearch_indices() {
+    local index_names=$1
+    local default_entry=${2:-'{"@timestamp":"0"}'}
+
+    for idx in $index_names; do
+        if ! so-elasticsearch-query "$idx" --fail --retry 3 --retry-delay 30 >/dev/null 2>&1; then
+            echo "Index does not already exist. Initializing $idx index."
+
+            if retry 3 10 "so-elasticsearch-query "$idx/_doc" -d '$default_entry' -XPOST --fail 2>/dev/null" '"successful":1'; then
+                echo "Successfully initialized $idx index."
+            else
+                echo "Failed to initialize $idx index after 3 attempts."
+            fi
+        else
+            echo "Index $idx already exists. No action needed."
+        fi
+    done
+}
+
 lookup_bond_interfaces() {
 	cat /proc/net/bonding/bond0 | grep "Slave Interface:" | sed -e "s/Slave Interface: //g"
 }
@@ -554,21 +568,39 @@ run_check_net_err() {
 }
 
 wait_for_salt_minion() {
-	local minion="$1"
-	local timeout="${2:-5}"
-	local logfile="${3:-'/dev/stdout'}"
-	retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$logfile" 2>&1 || fail
-	local attempt=0
-	# each attempts would take about 15 seconds
-	local maxAttempts=20
-	until check_salt_minion_status "$minion" "$timeout" "$logfile"; do
-		attempt=$((attempt+1))
-		if [[ $attempt -eq $maxAttempts ]]; then
-			return 1
-		fi
-		sleep 10
-	done
-	return 0
+    local minion="$1"
+    local max_wait="${2:-30}"
+    local interval="${3:-2}"
+    local logfile="${4:-'/dev/stdout'}"
+    local elapsed=0
+
+	echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Waiting for salt-minion '$minion' to be ready..."
+
+    while [ $elapsed -lt $max_wait ]; do
+        # Check if service is running
+		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Check if salt-minion service is running"
+        if ! systemctl is-active --quiet salt-minion; then
+            echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion service not running (elapsed: ${elapsed}s)"
+            sleep $interval
+            elapsed=$((elapsed + interval))
+            continue
+        fi
+		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion service is running"
+
+        # Check if minion responds to ping
+		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Check if $minion responds to ping"
+        if salt "$minion" test.ping --timeout=3 --out=json 2>> "$logfile" | grep -q "true"; then
+            echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion '$minion' is connected and ready!"
+            return 0
+        fi
+        
+        echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Waiting... (${elapsed}s / ${max_wait}s)"
+        sleep $interval
+        elapsed=$((elapsed + interval))
+    done
+
+    echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - ERROR: salt-minion '$minion' not ready after $max_wait seconds"
+    return 1
 }
 
 salt_minion_count() {
@@ -578,69 +610,19 @@ salt_minion_count() {
 }
 
 set_os() {
-	if [ -f /etc/redhat-release ]; then
-		if grep -q "Rocky Linux release 9" /etc/redhat-release; then
-			OS=rocky
-			OSVER=9
-			is_rocky=true
-			is_rpm=true
-		elif grep -q "CentOS Stream release 9" /etc/redhat-release; then
-			OS=centos
-			OSVER=9
-			is_centos=true
-			is_rpm=true
-		elif grep -q "AlmaLinux release 9" /etc/redhat-release; then
-			OS=alma
-			OSVER=9
-			is_alma=true
-			is_rpm=true
-		elif grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release; then
-			if [ -f /etc/oracle-release ]; then
-				OS=oracle
-				OSVER=9
-				is_oracle=true
-				is_rpm=true
-			else
-				OS=rhel
-				OSVER=9
-				is_rhel=true
-				is_rpm=true
-			fi
-		fi
-		cron_service_name="crond"
-	elif [ -f /etc/os-release ]; then
-		if grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then
-			OSVER=focal
-			UBVER=20.04
-			OS=ubuntu
-			is_ubuntu=true
-			is_deb=true
-		elif grep -q "UBUNTU_CODENAME=jammy" /etc/os-release; then
-			OSVER=jammy
-			UBVER=22.04
-			OS=ubuntu
-			is_ubuntu=true
-			is_deb=true
-		elif grep -q "VERSION_CODENAME=bookworm" /etc/os-release; then
-			OSVER=bookworm
-			DEBVER=12
-			is_debian=true
-			OS=debian
-			is_deb=true
-		fi
-		cron_service_name="cron"
+	if [ -f /etc/redhat-release ] && grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release && [ -f /etc/oracle-release ]; then
+		OS=oracle
+		OSVER=9
+		is_oracle=true
+		is_rpm=true
 	fi
+	cron_service_name="crond"
 }
 
 set_minionid() {
 	MINIONID=$(lookup_grain id)
 }
 
-set_palette() {
-	if [[ $is_deb ]]; then
-	    update-alternatives --set newt-palette /etc/newt/palette.original
-    fi
-}
 
 set_version() {
 	CURRENTVERSION=0.0.0

salt/common/tools/sbin/so-image-common

@@ -25,7 +25,6 @@ container_list() {
   if [ $MANAGERCHECK == 'so-import' ]; then
     TRUSTED_CONTAINERS=(
       "so-elasticsearch"
-      "so-idstools"
       "so-influxdb"
       "so-kibana"
       "so-kratos"
@@ -33,7 +32,6 @@ container_list() {
       "so-nginx"
       "so-pcaptools"
       "so-soc"
-      "so-steno"
       "so-suricata"
       "so-telegraf"
       "so-zeek"
@@ -49,7 +47,6 @@ container_list() {
       "so-elastic-fleet-package-registry"
       "so-elasticsearch"
       "so-idh"
-      "so-idstools"
       "so-influxdb"
       "so-kafka"
       "so-kibana"
@@ -60,7 +57,6 @@ container_list() {
       "so-pcaptools"
       "so-redis"
       "so-soc"
-      "so-steno"
       "so-strelka-backend"
       "so-strelka-manager"
       "so-suricata"
@@ -69,12 +65,10 @@ container_list() {
     )
   else
     TRUSTED_CONTAINERS=(
-      "so-idstools"
       "so-elasticsearch"
       "so-logstash"
       "so-nginx"
       "so-redis"
-      "so-steno"
       "so-suricata"
       "so-soc"
       "so-telegraf"

salt/common/tools/sbin/so-log-check

@@ -129,6 +129,8 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|responded with status-code 503" # telegraf getting 503 from ES during startup
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process_cluster_event_timeout_exception" # logstash waiting for elasticsearch to start
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not configured for GeoIP"     # SO does not bundle the maxminddb with Zeek
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|HTTP 404: Not Found"          # Salt loops until Kratos returns 200, during startup Kratos may not be ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -159,7 +161,9 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding ingest pipeline"       # false positive (elasticsearch ingest pipeline names contain 'error') 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating index template"      # false positive (elasticsearch index or template names contain 'error') 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating component template"  # false positive (elasticsearch index or template names contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading component template"  # false positive (elasticsearch index or template names contain 'error')
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -175,7 +179,6 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|salt-minion-check"            # bug in early 2.4 place Jinja script in non-jinja salt dir causing cron output errors
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|monitoring.metrics"           # known issue with elastic agent casting the field incorrectly if an integer value shows up before a float
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|repodownload.conf"            # known issue with reposync on pre-2.4.20 
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing versions record"      # stenographer corrupt index
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|soc.field."                   # known ingest type collisions issue with earlier versions of SO
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error parsing signature"      # Malformed Suricata rule, from upstream provider
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sticky buffer has no matches" # Non-critical Suricata error
@@ -223,6 +226,8 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint).*user so_kibana lacks the required permissions \[logs-\1" # Known issue with 3 integrations using kibana_system role vs creating unique api creds with proper permissions.
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
 fi
 
 RESULT=0

salt/common/tools/sbin/so-nsm-clear

@@ -55,19 +55,22 @@ if [ $SKIP -ne 1 ]; then
 fi
 
 delete_pcap() {
-  PCAP_DATA="/nsm/pcap/"
-  [ -d $PCAP_DATA ] && so-pcap-stop && rm -rf $PCAP_DATA/* && so-pcap-start
+  PCAP_DATA="/nsm/suripcap/"
+  [ -d $PCAP_DATA ] && rm -rf $PCAP_DATA/*
 }
 delete_suricata() {
   SURI_LOG="/nsm/suricata/"
-  [ -d $SURI_LOG ] && so-suricata-stop && rm -rf $SURI_LOG/* && so-suricata-start
+  [ -d $SURI_LOG ] && rm -rf $SURI_LOG/*
 }
 delete_zeek() {
   ZEEK_LOG="/nsm/zeek/logs/"
   [ -d $ZEEK_LOG ] && so-zeek-stop && rm -rf $ZEEK_LOG/* && so-zeek-start
 }
 
+so-suricata-stop
 delete_pcap
 delete_suricata
 delete_zeek
+so-suricata-start
+
 

salt/common/tools/sbin/so-restart

@@ -23,7 +23,6 @@ if [ $# -ge 1 ]; then
         fi
 
         case $1 in
-                "steno") docker stop so-steno && docker rm so-steno && salt-call state.apply pcap queue=True;;
                 "elastic-fleet") docker stop so-elastic-fleet && docker rm so-elastic-fleet && salt-call state.apply elasticfleet queue=True;;
                 *)  docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
         esac

salt/common/tools/sbin/so-sensor-clean

@@ -72,7 +72,7 @@ clean() {
 		done
 	fi
 
-	## Clean up extracted pcaps from Steno
+	## Clean up extracted pcaps
 	PCAPS='/nsm/pcapout'
 	OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
 	if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then

salt/common/tools/sbin/so-start

@@ -23,7 +23,6 @@ if [ $# -ge 1 ]; then
 
 	case $1 in
    		"all") salt-call state.highstate queue=True;;
-   		"steno") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply pcap queue=True; fi ;; 
    		"elastic-fleet") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply elasticfleet queue=True; fi ;; 
    		*) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; 
 	esac

salt/common/tools/sbin_jinja/so-desktop-install

@@ -6,7 +6,7 @@
 # Elastic License 2.0.
 
 source /usr/sbin/so-common
-doc_desktop_url="$DOC_BASE_URL/desktop.html"
+doc_desktop_url="$DOC_BASE_URL/desktop"
 
 {# we only want the script to install the desktop if it is OEL -#}
 {% if grains.os == 'OEL' -%}

salt/common/tools/sbin_jinja/so-import-pcap

@@ -85,7 +85,7 @@ function suricata() {
   docker run --rm \
     -v /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro \
     -v /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro \
-    -v /opt/so/conf/suricata/rules:/etc/suricata/rules:ro \
+    -v /opt/so/rules/suricata/:/etc/suricata/rules:ro \
     -v ${LOG_PATH}:/var/log/suricata/:rw \
     -v ${NSM_PATH}/:/nsm/:rw \
     -v "$PCAP:/input.pcap:ro" \

salt/curator/disabled.sls

@@ -1,34 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-so-curator:
-  docker_container.absent:
-    - force: True
-
-so-curator_so-status.disabled:
-  file.line:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - match: ^so-curator$
-    - mode: delete
-
-so-curator-cluster-close:
-  cron.absent:
-    - identifier: so-curator-cluster-close
-
-so-curator-cluster-delete:
-  cron.absent:
-    - identifier: so-curator-cluster-delete
-
-delete_curator_configuration:
-  file.absent:
-    - name: /opt/so/conf/curator
-    - recurse: True
-
-{% set files = salt.file.find(path='/usr/sbin', name='so-curator*') %}
-{% if files|length > 0 %}
-delete_curator_scripts:
-  file.absent:
-    - names: {{files|yaml}}
-{% endif %}

salt/desktop/trusted-ca.sls

@@ -3,29 +3,16 @@
 {# we only want this state to run it is CentOS #}
 {% if GLOBALS.os == 'OEL' %}
 
-  {% set global_ca_text = [] %}
-  {% set global_ca_server = [] %}
-  {% set manager = GLOBALS.manager %}
-  {% set x509dict = salt['mine.get'](manager | lower~'*', 'x509.get_pem_entries') %}
-    {% for host in x509dict %}
-      {% if host.split('_')|last in ['manager', 'managersearch', 'standalone', 'import', 'eval'] %}
-        {% do global_ca_text.append(x509dict[host].get('/etc/pki/ca.crt')|replace('\n', '')) %}
-        {% do global_ca_server.append(host) %}
-      {% endif %}
-    {% endfor %}
-  {% set trusttheca_text = global_ca_text[0] %}
-  {% set ca_server = global_ca_server[0] %}
-
 trusted_ca:
-  x509.pem_managed:
+  file.managed:
     - name: /etc/pki/ca-trust/source/anchors/ca.crt
-    - text:  {{ trusttheca_text }}
+    - source: salt://ca/files/ca.crt
 
 update_ca_certs:
   cmd.run:
     - name: update-ca-trust
     - onchanges:
-      - x509: trusted_ca
+      - file: trusted_ca
 
 {% else %}
 

salt/docker/defaults.yaml

@@ -1,6 +1,10 @@
 docker:
   range: '172.17.1.0/24'
   gateway: '172.17.1.1'
+  ulimits:
+    - name: nofile
+      soft: 1048576
+      hard: 1048576
   containers:
     'so-dockerregistry':
       final_octet: 20
@@ -9,6 +13,7 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-elastic-fleet':
       final_octet: 21
       port_bindings:
@@ -16,6 +21,7 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-elasticsearch':
       final_octet: 22
       port_bindings:
@@ -24,11 +30,16 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
-    'so-idstools':
-      final_octet: 25
-      custom_bind_mounts: []
-      extra_hosts: []
-      extra_env: []
+      ulimits:
+        - name: memlock
+          soft: -1
+          hard: -1
+        - name: nofile
+          soft: 65536
+          hard: 65536
+        - name: nproc
+          soft: 4096
+          hard: 4096
     'so-influxdb':
       final_octet: 26
       port_bindings:
@@ -36,6 +47,7 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-kibana':
       final_octet: 27
       port_bindings:
@@ -43,6 +55,7 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-kratos':
       final_octet: 28
       port_bindings:
@@ -51,6 +64,7 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-hydra':
       final_octet: 30
       port_bindings:
@@ -59,6 +73,7 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-logstash':
       final_octet: 29
       port_bindings:
@@ -75,6 +90,7 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-nginx':
       final_octet: 31
       port_bindings:
@@ -86,6 +102,7 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-nginx-fleet-node':
       final_octet: 31
       port_bindings:
@@ -93,6 +110,7 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-redis':
       final_octet: 33
       port_bindings:
@@ -101,11 +119,13 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-sensoroni':
       final_octet: 99
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-soc':
       final_octet: 34
       port_bindings:
@@ -113,16 +133,19 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-strelka-backend':
       final_octet: 36
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-strelka-filestream':
       final_octet: 37
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-strelka-frontend':
       final_octet: 38
       port_bindings:
@@ -130,11 +153,13 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-strelka-manager':
       final_octet: 39
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-strelka-gatekeeper':
       final_octet: 40
       port_bindings:
@@ -142,6 +167,7 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-strelka-coordinator':
       final_octet: 41
       port_bindings:
@@ -149,11 +175,13 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-elastalert':
       final_octet: 42
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-elastic-fleet-package-registry':
       final_octet: 44
       port_bindings:
@@ -161,11 +189,13 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-idh':
       final_octet: 45
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-elastic-agent':
       final_octet: 46
       port_bindings:
@@ -174,28 +204,34 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []
     'so-telegraf':
       final_octet: 99
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
-    'so-steno':
-      final_octet: 99
-      custom_bind_mounts: []
-      extra_hosts: []
-      extra_env: []
+      ulimits: []
     'so-suricata':
       final_octet: 99
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
       ulimits:
-        - memlock=524288000
+        - name: memlock
+          soft: 524288000
+          hard: 524288000
     'so-zeek':
       final_octet: 99
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits:
+        - name: core
+          soft: 0
+          hard: 0
+        - name: nofile
+          soft: 1048576
+          hard: 1048576
     'so-kafka':
       final_octet: 88
       port_bindings:
@@ -206,3 +242,4 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+      ulimits: []

salt/docker/docker.map.jinja

@@ -1,8 +1,8 @@
 {% import_yaml 'docker/defaults.yaml' as DOCKERDEFAULTS %}
-{% set DOCKER = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %}
-{% set RANGESPLIT = DOCKER.range.split('.') %}
+{% set DOCKERMERGED = salt['pillar.get']('docker', DOCKERDEFAULTS.docker, merge=True) %}
+{% set RANGESPLIT = DOCKERMERGED.range.split('.') %}
 {% set FIRSTTHREE = RANGESPLIT[0] ~ '.' ~ RANGESPLIT[1] ~ '.' ~ RANGESPLIT[2] ~ '.' %}
 
-{% for container, vals in DOCKER.containers.items() %}
-{%   do DOCKER.containers[container].update({'ip': FIRSTTHREE ~ DOCKER.containers[container].final_octet}) %}
+{% for container, vals in DOCKERMERGED.containers.items() %}
+{%   do DOCKERMERGED.containers[container].update({'ip': FIRSTTHREE ~ DOCKERMERGED.containers[container].final_octet}) %}
 {% endfor %}

salt/docker/files/daemon.json.jinja

@@ -0,0 +1,24 @@
+{% from 'docker/docker.map.jinja' import DOCKERMERGED -%}
+{
+  "registry-mirrors": [
+    "https://:5000"
+  ],
+  "bip": "172.17.0.1/24",
+  "default-address-pools": [
+    {
+      "base": "172.17.0.0/24",
+      "size": 24
+    }
+  ]
+{%- if DOCKERMERGED.ulimits %},
+  "default-ulimits": {
+{%-   for ULIMIT in DOCKERMERGED.ulimits %}
+    "{{ ULIMIT.name }}": {
+      "Name": "{{ ULIMIT.name }}",
+      "Soft": {{ ULIMIT.soft }},
+      "Hard": {{ ULIMIT.hard }}
+    }{{ "," if not loop.last else "" }}
+{%-   endfor %}
+  }
+{%- endif %}
+}

salt/docker/init.sls

@@ -3,61 +3,27 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{% from 'docker/docker.map.jinja' import DOCKER %}
+{% from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
-# include ssl since docker service requires the intca
+# docker service requires the ca.crt
 include:
-  - ssl
+  - ca
 
 dockergroup:
   group.present:
     - name: docker
     - gid: 920
 
-{% if GLOBALS.os_family == 'Debian' %}
-{%    if grains.oscodename == 'bookworm' %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.7.21-1
-      - docker-ce: 5:27.2.0-1~debian.12~bookworm
-      - docker-ce-cli: 5:27.2.0-1~debian.12~bookworm
-      - docker-ce-rootless-extras: 5:27.2.0-1~debian.12~bookworm
+      - containerd.io: 2.2.1-1.el9
+      - docker-ce: 3:29.2.1-1.el9
+      - docker-ce-cli: 1:29.2.1-1.el9
+      - docker-ce-rootless-extras: 29.2.1-1.el9
     - hold: True
     - update_holds: True
-{%    elif grains.oscodename == 'jammy' %}
-dockerheldpackages:
-  pkg.installed:
-    - pkgs:
-      - containerd.io: 1.7.21-1
-      - docker-ce: 5:27.2.0-1~ubuntu.22.04~jammy
-      - docker-ce-cli: 5:27.2.0-1~ubuntu.22.04~jammy
-      - docker-ce-rootless-extras: 5:27.2.0-1~ubuntu.22.04~jammy
-    - hold: True
-    - update_holds: True
-{%    else %}
-dockerheldpackages:
-  pkg.installed:
-    - pkgs:
-      - containerd.io: 1.7.21-1
-      - docker-ce: 5:27.2.0-1~ubuntu.20.04~focal
-      - docker-ce-cli: 5:27.2.0-1~ubuntu.20.04~focal
-      - docker-ce-rootless-extras: 5:27.2.0-1~ubuntu.20.04~focal
-    - hold: True
-    - update_holds: True
-{%   endif %}
-{% else %}
-dockerheldpackages:
-  pkg.installed:
-    - pkgs:
-      - containerd.io: 1.7.21-3.1.el9
-      - docker-ce: 3:27.2.0-1.el9
-      - docker-ce-cli: 1:27.2.0-1.el9
-      - docker-ce-rootless-extras: 27.2.0-1.el9
-    - hold: True
-    - update_holds: True
-{% endif %}
 
 #disable docker from managing iptables
 iptables_disabled:
@@ -75,10 +41,9 @@ dockeretc:
   file.directory:
     - name: /etc/docker
 
-# Manager daemon.json
 docker_daemon:
   file.managed:
-    - source: salt://common/files/daemon.json
+    - source: salt://docker/files/daemon.json.jinja
     - name: /etc/docker/daemon.json
     - template: jinja 
 
@@ -89,10 +54,9 @@ docker_running:
     - enable: True
     - watch:
       - file: docker_daemon
-      - x509: trusttheca
     - require:
       - file: docker_daemon
-      - x509: trusttheca
+      - file: trusttheca
 
 
 # Reserve OS ports for Docker proxy in case boot settings are not already applied/present
@@ -110,12 +74,12 @@ dockerreserveports:
 sos_docker_net:
   docker_network.present:
     - name: sobridge
-    - subnet: {{ DOCKER.range }}
-    - gateway: {{ DOCKER.gateway }}
+    - subnet: {{ DOCKERMERGED.range }}
+    - gateway: {{ DOCKERMERGED.gateway }}
     - options:
         com.docker.network.bridge.name: 'sobridge'
         com.docker.network.driver.mtu: '1500'
         com.docker.network.bridge.enable_ip_masquerade: 'true'
         com.docker.network.bridge.enable_icc: 'true'
         com.docker.network.bridge.host_binding_ipv4: '0.0.0.0'
-    - unless: 'docker network ls | grep sobridge'
+    - unless: ip l | grep sobridge

salt/docker/soc_docker.yaml

@@ -7,6 +7,25 @@ docker:
     description: Default docker IP range for containers.
     helpLink: docker.html
     advanced: True
+  ulimits:
+    description: |
+      Default ulimit settings applied to all containers via the Docker daemon. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with soft and hard limits. Individual container ulimits override these defaults. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime.
+    forcedType: "[]{}"
+    syntax: json
+    advanced: True
+    helpLink: docker.html
+    uiElements:
+    - field: name
+      label: Resource Name
+      required: True
+      regex: ^(cpu|fsize|data|stack|core|rss|nproc|nofile|memlock|as|locks|sigpending|msgqueue|nice|rtprio|rttime)$
+      regexFailureMessage: You must enter a valid ulimit name (cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime).
+    - field: soft
+      label: Soft Limit
+      forcedType: int
+    - field: hard
+      label: Hard Limit
+      forcedType: int
   containers:
     so-dockerregistry: &dockerOptions
       final_octet:
@@ -39,9 +58,27 @@ docker:
         helpLink: docker.html
         multiline: True
         forcedType: "[]string"
+      ulimits:
+        description: |
+          Ulimit settings for the container. Each entry specifies a resource name (e.g. nofile, memlock, core, nproc) with optional soft and hard limits. Valid resource names include: cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime.
+        advanced: True
+        helpLink: docker.html
+        forcedType: "[]{}"
+        syntax: json
+        uiElements:
+        - field: name
+          label: Resource Name
+          required: True
+          regex: ^(cpu|fsize|data|stack|core|rss|nproc|nofile|memlock|as|locks|sigpending|msgqueue|nice|rtprio|rttime)$
+          regexFailureMessage: You must enter a valid ulimit name (cpu, fsize, data, stack, core, rss, nproc, nofile, memlock, as, locks, sigpending, msgqueue, nice, rtprio, rttime).
+        - field: soft
+          label: Soft Limit
+          forcedType: int
+        - field: hard
+          label: Hard Limit
+          forcedType: int
     so-elastic-fleet: *dockerOptions
     so-elasticsearch: *dockerOptions
-    so-idstools: *dockerOptions
     so-influxdb: *dockerOptions
     so-kibana: *dockerOptions
     so-kratos: *dockerOptions
@@ -63,43 +100,6 @@ docker:
     so-idh: *dockerOptions
     so-elastic-agent: *dockerOptions
     so-telegraf: *dockerOptions
-    so-steno: *dockerOptions
-    so-suricata:
-      final_octet:
-        description: Last octet of the container IP address.
-        helpLink: docker.html
-        readonly: True
-        advanced: True
-        global: True
-      port_bindings:
-        description: List of port bindings for the container.
-        helpLink: docker.html
-        advanced: True
-        multiline: True
-        forcedType: "[]string"
-      custom_bind_mounts:
-        description: List of custom local volume bindings.
-        advanced: True
-        helpLink: docker.html
-        multiline: True
-        forcedType: "[]string"
-      extra_hosts:
-        description: List of additional host entries for the container.
-        advanced: True
-        helpLink: docker.html
-        multiline: True
-        forcedType: "[]string"
-      extra_env:
-        description: List of additional ENV entries for the container.
-        advanced: True
-        helpLink: docker.html
-        multiline: True
-        forcedType: "[]string"
-      ulimits:
-        description: Ulimits for the container, in bytes.
-        advanced: True
-        helpLink: docker.html
-        multiline: True
-        forcedType: "[]string"
+    so-suricata: *dockerOptions
     so-zeek: *dockerOptions
-    so-kafka: *dockerOptions
+    so-kafka: *dockerOptions

salt/elastalert/enabled.sls

@@ -6,7 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKER %}
+{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 
 include:
   - elastalert.config
@@ -24,7 +24,7 @@ so-elastalert:
     - user: so-elastalert
     - networks:
       - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-elastalert'].ip }}
+        - ipv4_address: {{ DOCKERMERGED.containers['so-elastalert'].ip }}
     - detach: True
     - binds:
       - /opt/so/rules/elastalert:/opt/elastalert/rules/:ro
@@ -33,24 +33,30 @@ so-elastalert:
       - /opt/so/conf/elastalert/predefined/:/opt/elastalert/predefined/:ro
       - /opt/so/conf/elastalert/custom/:/opt/elastalert/custom/:ro
       - /opt/so/conf/elastalert/elastalert_config.yaml:/opt/elastalert/config.yaml:ro
-      {% if DOCKER.containers['so-elastalert'].custom_bind_mounts %}
-        {% for BIND in DOCKER.containers['so-elastalert'].custom_bind_mounts %}
+      {% if DOCKERMERGED.containers['so-elastalert'].custom_bind_mounts %}
+        {% for BIND in DOCKERMERGED.containers['so-elastalert'].custom_bind_mounts %}
       - {{ BIND }}
         {% endfor %}
       {% endif %}
     - extra_hosts:
       - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
-      {% if DOCKER.containers['so-elastalert'].extra_hosts %}
-        {% for XTRAHOST in DOCKER.containers['so-elastalert'].extra_hosts %}
+      {% if DOCKERMERGED.containers['so-elastalert'].extra_hosts %}
+        {% for XTRAHOST in DOCKERMERGED.containers['so-elastalert'].extra_hosts %}
       - {{ XTRAHOST }}
         {% endfor %}
       {% endif %}
-      {% if DOCKER.containers['so-elastalert'].extra_env %}
+      {% if DOCKERMERGED.containers['so-elastalert'].extra_env %}
     - environment:
-        {% for XTRAENV in DOCKER.containers['so-elastalert'].extra_env %}
+        {% for XTRAENV in DOCKERMERGED.containers['so-elastalert'].extra_env %}
       - {{ XTRAENV }}
         {% endfor %}
       {% endif %}
+    {% if DOCKERMERGED.containers['so-elastalert'].ulimits %}
+    - ulimits:
+    {%   for ULIMIT in DOCKERMERGED.containers['so-elastalert'].ulimits %}
+      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
+    {%   endfor %}
+    {% endif %}
     - require:
       - cmd: wait_for_elasticsearch
       - file: elastarules
@@ -60,7 +66,7 @@ so-elastalert:
     - watch:
       - file: elastaconf
     - onlyif:
-      - "so-elasticsearch-query / | jq -r '.version.number[0:1]' | grep -q 8" {# only run this state if elasticsearch is version 8 #}
+      - "so-elasticsearch-query / | jq -r '.version.number[0:1]' | grep -q 9" {# only run this state if elasticsearch is version 9 #}
 
 delete_so-elastalert_so-status.disabled:
   file.uncomment:

salt/elastalert/files/custom/placeholder

@@ -1 +0,0 @@
-THIS IS A PLACEHOLDER FILE

salt/elastic-fleet-package-registry/enabled.sls

@@ -6,7 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKER %}
+{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 
 include:
   - elastic-fleet-package-registry.config
@@ -21,30 +21,36 @@ so-elastic-fleet-package-registry:
     - user: 948
     - networks:
       - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-elastic-fleet-package-registry'].ip }}
+        - ipv4_address: {{ DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ip }}
     - extra_hosts:
         - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }}
-        {% if DOCKER.containers['so-elastic-fleet-package-registry'].extra_hosts %}
-          {% for XTRAHOST in DOCKER.containers['so-elastic-fleet-package-registry'].extra_hosts %}
+        {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_hosts %}
+          {% for XTRAHOST in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_hosts %}
         - {{ XTRAHOST }}
           {% endfor %}
         {% endif %}
     - port_bindings:
-      {% for BINDING in DOCKER.containers['so-elastic-fleet-package-registry'].port_bindings %}
+      {% for BINDING in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].port_bindings %}
       - {{ BINDING }}
       {% endfor %}
-    {% if DOCKER.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
+    {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
     - binds:
-        {% for BIND in DOCKER.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
+        {% for BIND in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].custom_bind_mounts %}
       - {{ BIND }}
         {% endfor %}
     {% endif %}
-    {% if DOCKER.containers['so-elastic-fleet-package-registry'].extra_env %}
+    {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_env %}
     - environment:
-        {% for XTRAENV in DOCKER.containers['so-elastic-fleet-package-registry'].extra_env %}
+        {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].extra_env %}
       - {{ XTRAENV }}
         {% endfor %}
       {% endif %}
+    {% if DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ulimits %}
+    - ulimits:
+    {%   for ULIMIT in DOCKERMERGED.containers['so-elastic-fleet-package-registry'].ulimits %}
+      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
+    {%   endfor %}
+    {% endif %}
 delete_so-elastic-fleet-package-registry_so-status.disabled:
   file.uncomment:
     - name: /opt/so/conf/so-status/so-status.conf

salt/elasticagent/enabled.sls

@@ -6,9 +6,10 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKER %}
+{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 
 include:
+  - ca
   - elasticagent.config
   - elasticagent.sostatus
 
@@ -21,17 +22,17 @@ so-elastic-agent:
     - user: 949
     - networks:
       - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-elastic-agent'].ip }}
+        - ipv4_address: {{ DOCKERMERGED.containers['so-elastic-agent'].ip }}
     - extra_hosts:
         - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
         - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }}
-        {% if DOCKER.containers['so-elastic-agent'].extra_hosts %}
-          {% for XTRAHOST in DOCKER.containers['so-elastic-agent'].extra_hosts %}
+        {% if DOCKERMERGED.containers['so-elastic-agent'].extra_hosts %}
+          {% for XTRAHOST in DOCKERMERGED.containers['so-elastic-agent'].extra_hosts %}
         - {{ XTRAHOST }}
           {% endfor %}
         {% endif %}
     - port_bindings:
-      {% for BINDING in DOCKER.containers['so-elastic-agent'].port_bindings %}
+      {% for BINDING in DOCKERMERGED.containers['so-elastic-agent'].port_bindings %}
       - {{ BINDING }}
       {% endfor %}
     - binds:
@@ -40,23 +41,31 @@ so-elastic-agent:
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro 
       - /nsm:/nsm:ro
       - /opt/so/log:/opt/so/log:ro
-     {% if DOCKER.containers['so-elastic-agent'].custom_bind_mounts %}
-        {% for BIND in DOCKER.containers['so-elastic-agent'].custom_bind_mounts %}
+     {% if DOCKERMERGED.containers['so-elastic-agent'].custom_bind_mounts %}
+        {% for BIND in DOCKERMERGED.containers['so-elastic-agent'].custom_bind_mounts %}
       - {{ BIND }}
         {% endfor %}
       {% endif %}
     - environment:
       - FLEET_CA=/etc/pki/tls/certs/intca.crt
       - LOGS_PATH=logs
-      {% if DOCKER.containers['so-elastic-agent'].extra_env %}
-        {% for XTRAENV in DOCKER.containers['so-elastic-agent'].extra_env %}
+      {% if DOCKERMERGED.containers['so-elastic-agent'].extra_env %}
+        {% for XTRAENV in DOCKERMERGED.containers['so-elastic-agent'].extra_env %}
       - {{ XTRAENV }}
         {% endfor %}
       {% endif %}
+    {% if DOCKERMERGED.containers['so-elastic-agent'].ulimits %}
+    - ulimits:
+    {%   for ULIMIT in DOCKERMERGED.containers['so-elastic-agent'].ulimits %}
+      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
+    {%   endfor %}
+    {% endif %}
     - require:
       - file: create-elastic-agent-config
+      - file: trusttheca
     - watch:
       - file: create-elastic-agent-config
+      - file: trusttheca
 
 delete_so-elastic-agent_so-status.disabled:
   file.uncomment:

salt/elasticagent/files/elastic-agent.yml.jinja

@@ -3,7 +3,7 @@
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 
 id: aea1ba80-1065-11ee-a369-97538913b6a9
-revision: 1
+revision: 4
 outputs:
   default:
     type: elasticsearch
@@ -22,242 +22,133 @@ agent:
     metrics: false
   features: {}
 inputs:
-  - id: logfile-logs-fefef78c-422f-4cfa-8abf-4cd1b9428f62
-    name: import-evtx-logs
-    revision: 2
-    type: logfile
-    use_output: default
-    meta:
-      package:
-        name: log
-        version:
-    data_stream:
-      namespace: so
-    package_policy_id: fefef78c-422f-4cfa-8abf-4cd1b9428f62
-    streams:
-      - id: logfile-log.log-fefef78c-422f-4cfa-8abf-4cd1b9428f62
-        data_stream:
-          dataset: import
-        paths:
-          - /nsm/import/*/evtx/*.json
-        processors:
-          - dissect:
-              field: log.file.path
-              tokenizer: '/nsm/import/%{import.id}/evtx/%{import.file}'
-              target_prefix: ''
-          - decode_json_fields:
-              fields:
-                - message
-              target: ''
-          - drop_fields:
-              ignore_missing: true
-              fields:
-                - host
-          - add_fields:
-              fields:
-                dataset: system.security
-                type: logs
-                namespace: default
-              target: data_stream
-          - add_fields:
-              fields:
-                dataset: system.security
-                module: system
-                imported: true
-              target: event
-          - then:
-              - add_fields:
-                  fields:
-                    dataset: windows.sysmon_operational
-                  target: data_stream
-              - add_fields:
-                  fields:
-                    dataset: windows.sysmon_operational
-                    module: windows
-                    imported: true
-                  target: event
-            if:
-              equals:
-                winlog.channel: Microsoft-Windows-Sysmon/Operational
-          - then:
-              - add_fields:
-                  fields:
-                    dataset: system.application
-                  target: data_stream
-              - add_fields:
-                  fields:
-                    dataset: system.application
-                  target: event
-            if:
-              equals:
-                winlog.channel: Application
-          - then:
-              - add_fields:
-                  fields:
-                    dataset: system.system
-                  target: data_stream
-              - add_fields:
-                  fields:
-                    dataset: system.system
-                  target: event
-            if:
-              equals:
-                winlog.channel: System
-          - then:
-              - add_fields:
-                  fields:
-                    dataset: windows.powershell_operational
-                  target: data_stream
-              - add_fields:
-                  fields:
-                    dataset: windows.powershell_operational
-                    module: windows
-                  target: event
-            if:
-              equals:
-                winlog.channel: Microsoft-Windows-PowerShell/Operational
-        tags:
-          - import
-  - id: logfile-redis-fc98c947-7d17-4861-a318-7ad075f6d1b0
-    name: redis-logs
-    revision: 2
-    type: logfile
-    use_output: default
-    meta:
-      package:
-        name: redis
-        version:
-    data_stream:
-      namespace: default
-    package_policy_id: fc98c947-7d17-4861-a318-7ad075f6d1b0
-    streams:
-      - id: logfile-redis.log-fc98c947-7d17-4861-a318-7ad075f6d1b0
-        data_stream:
-          dataset: redis.log
-          type: logs
-        exclude_files:
-          - .gz$
-        paths:
-          - /opt/so/log/redis/redis.log
-        tags:
-          - redis-log
-        exclude_lines:
-          - '^\s+[\-`(''.|_]'
-  - id: logfile-logs-3b56803d-5ade-4c93-b25e-9b37182f66b8
+  - id: filestream-filestream-85820eb0-25ef-11f0-a18d-1b26f69b8310
     name: import-suricata-logs
-    revision: 2
-    type: logfile
+    revision: 3
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: 3b56803d-5ade-4c93-b25e-9b37182f66b8
+    package_policy_id: 85820eb0-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-3b56803d-5ade-4c93-b25e-9b37182f66b8
+      - id: filestream-filestream.generic-85820eb0-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: import
-        pipeline: suricata.common
         paths:
           - /nsm/import/*/suricata/eve*.json
+        pipeline: suricata.common
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - \.gz$
+        ignore_older: 72h
+        clean_inactive: -1
+        parsers: null
         processors:
           - add_fields:
+              target: event
               fields:
+                category: network
                 module: suricata
                 imported: true
-                category: network
-              target: event
           - dissect:
+              tokenizer: /nsm/import/%{import.id}/suricata/%{import.file}
               field: log.file.path
-              tokenizer: '/nsm/import/%{import.id}/suricata/%{import.file}'
               target_prefix: ''
-  - id: logfile-logs-c327e1a3-1ebe-449c-a8eb-f6f35032e69d
-    name: soc-server-logs
-    revision: 2
-    type: logfile
+        file_identity.native: null
+        prospector.scanner.fingerprint.enabled: false
+  - id: filestream-filestream-86b4e960-25ef-11f0-a18d-1b26f69b8310
+    name: import-zeek-logs
+    revision: 3
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+    package_policy_id: 86b4e960-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+      - id: filestream-filestream.generic-86b4e960-25ef-11f0-a18d-1b26f69b8310
         data_stream:
-          dataset: soc
-        pipeline: common
+          dataset: import
         paths:
-          - /opt/so/log/soc/sensoroni-server.log
+          - /nsm/import/*/zeek/logs/*.log
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - >-
+            (broker|capture_loss|cluster|conn-summary|console|ecat_arp_info|known_certs|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout).log$
+        clean_inactive: -1
+        parsers: null
         processors:
-          - decode_json_fields:
-              add_error_key: true
-              process_array: true
-              max_depth: 2
-              fields:
-                - message
-              target: soc
+          - dissect:
+              tokenizer: /nsm/import/%{import.id}/zeek/logs/%{import.file}
+              field: log.file.path
+              target_prefix: ''
+          - script:
+              lang: javascript
+              source: |
+                function process(event) {
+                  var pl = event.Get("import.file").slice(0,-4);
+                  event.Put("@metadata.pipeline", "zeek." + pl);
+                }
           - add_fields:
-              fields:
-                module: soc
-                dataset_temp: server
-                category: host
               target: event
-          - rename:
-              ignore_missing: true
               fields:
-                - from: soc.fields.sourceIp
-                  to: source.ip
-                - from: soc.fields.status
-                  to: http.response.status_code
-                - from: soc.fields.method
-                  to: http.request.method
-                - from: soc.fields.path
-                  to: url.path
-                - from: soc.message
-                  to: event.action
-                - from: soc.level
-                  to: log.level
-        tags:
-          - so-soc
-  - id: logfile-logs-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+                category: network
+                module: zeek
+                imported: true
+          - add_tags:
+              tags: ics
+              when:
+                regexp:
+                  import.file: >-
+                    ^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*
+        file_identity.native: null
+        prospector.scanner.fingerprint.enabled: false
+  - id: filestream-filestream-91741240-25ef-11f0-a18d-1b26f69b8310
     name: soc-sensoroni-logs
-    revision: 2
-    type: logfile
+    revision: 3
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: 906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+    package_policy_id: 91741240-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+      - id: filestream-filestream.generic-91741240-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: soc
-        pipeline: common
         paths:
           - /opt/so/log/sensoroni/sensoroni.log
+        pipeline: common
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - \.gz$
+        clean_inactive: -1
+        parsers: null
         processors:
           - decode_json_fields:
-              add_error_key: true
-              process_array: true
-              max_depth: 2
               fields:
                 - message
               target: sensoroni
+              process_array: true
+              max_depth: 2
+              add_error_key: true
           - add_fields:
+              target: event
               fields:
+                category: host
                 module: soc
                 dataset_temp: sensoroni
-                category: host
-              target: event
           - rename:
-              ignore_missing: true
               fields:
                 - from: sensoroni.fields.sourceIp
                   to: source.ip
@@ -271,141 +162,100 @@ inputs:
                   to: event.action
                 - from: sensoroni.level
                   to: log.level
-  - id: logfile-logs-df0d7f2c-221f-433b-b18b-d1cf83250515
-    name: soc-salt-relay-logs
-    revision: 2
-    type: logfile
-    use_output: default
-    meta:
-      package:
-        name: log
-        version:
-    data_stream:
-      namespace: so
-    package_policy_id: df0d7f2c-221f-433b-b18b-d1cf83250515
-    streams:
-      - id: logfile-log.log-df0d7f2c-221f-433b-b18b-d1cf83250515
-        data_stream:
-          dataset: soc
-        pipeline: common
-        paths:
-          - /opt/so/log/soc/salt-relay.log
-        processors:
-          - dissect:
-              field: message
-              tokenizer: '%{soc.ts} | %{event.action}'
-              target_prefix: ''
-          - add_fields:
-              fields:
-                module: soc
-                dataset_temp: salt_relay
-                category: host
-              target: event
-        tags:
-          - so-soc
-  - id: logfile-logs-74bd2366-fe52-493c-bddc-843a017fc4d0
-    name: soc-auth-sync-logs
-    revision: 2
-    type: logfile
-    use_output: default
-    meta:
-      package:
-        name: log
-        version:
-    data_stream:
-      namespace: so
-    package_policy_id: 74bd2366-fe52-493c-bddc-843a017fc4d0
-    streams:
-      - id: logfile-log.log-74bd2366-fe52-493c-bddc-843a017fc4d0
-        data_stream:
-          dataset: soc
-        pipeline: common
-        paths:
-          - /opt/so/log/soc/sync.log
-        processors:
-          - dissect:
-              field: message
-              tokenizer: '%{event.action}'
-              target_prefix: ''
-          - add_fields:
-              fields:
-                module: soc
-                dataset_temp: auth_sync
-                category: host
-              target: event
-        tags:
-          - so-soc
-  - id: logfile-logs-d151d9bf-ff2a-4529-9520-c99244bc0253
+              ignore_missing: true
+        file_identity.native: null
+        prospector.scanner.fingerprint.enabled: false
+  - id: filestream-filestream-976e3900-25ef-11f0-a18d-1b26f69b8310
     name: suricata-logs
-    revision: 2
-    type: logfile
+    revision: 3
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: d151d9bf-ff2a-4529-9520-c99244bc0253
+    package_policy_id: 976e3900-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-d151d9bf-ff2a-4529-9520-c99244bc0253
+      - id: filestream-filestream.generic-976e3900-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: suricata
-        pipeline: suricata.common
         paths:
           - /nsm/suricata/eve*.json
+        pipeline: suricata.common
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - \.gz$
+        clean_inactive: -1
+        parsers: null
         processors:
           - add_fields:
-              fields:
-                module: suricata
-                category: network
               target: event
-  - id: logfile-logs-31f94d05-ae75-40ee-b9c5-0e0356eff327
+              fields:
+                category: network
+                module: suricata
+        file_identity.native: null
+        prospector.scanner.fingerprint.enabled: false
+  - id: filestream-filestream-95091fe0-25ef-11f0-a18d-1b26f69b8310
     name: strelka-logs
-    revision: 2
-    type: logfile
+    revision: 3
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: 31f94d05-ae75-40ee-b9c5-0e0356eff327
+    package_policy_id: 95091fe0-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-31f94d05-ae75-40ee-b9c5-0e0356eff327
+      - id: filestream-filestream.generic-95091fe0-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: strelka
-        pipeline: strelka.file
         paths:
           - /nsm/strelka/log/strelka.log
+        pipeline: strelka.file
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - \.gz$
+        clean_inactive: -1
+        parsers: null
         processors:
           - add_fields:
-              fields:
-                module: strelka
-                category: file
               target: event
-  - id: logfile-logs-6197fe84-9b58-4d9b-8464-3d517f28808d
+              fields:
+                category: file
+                module: strelka
+        file_identity.native: null
+        prospector.scanner.fingerprint.enabled: false
+  - id: filestream-filestream-9f309ca0-25ef-11f0-a18d-1b26f69b8310
     name: zeek-logs
-    revision: 1
-    type: logfile
+    revision: 2
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
-        version: 
+        name: filestream
+        version:
     data_stream:
       namespace: so
-    package_policy_id: 6197fe84-9b58-4d9b-8464-3d517f28808d
+    package_policy_id: 9f309ca0-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-6197fe84-9b58-4d9b-8464-3d517f28808d
+      - id: filestream-filestream.generic-9f309ca0-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: zeek
         paths:
           - /nsm/zeek/logs/current/*.log
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - >-
+            (broker|capture_loss|cluster|conn-summary|console|ecat_arp_info|known_certs|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout).log$
+        clean_inactive: -1
+        parsers: null
         processors:
           - dissect:
-              tokenizer: '/nsm/zeek/logs/current/%{pipeline}.log'
+              tokenizer: /nsm/zeek/logs/current/%{pipeline}.log
               field: log.file.path
               trim_chars: .log
               target_prefix: ''
@@ -427,18 +277,17 @@ inputs:
                 regexp:
                   pipeline: >-
                     ^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*
-        exclude_files:
-          - >-
-            broker|capture_loss|cluster|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout.log$
+        file_identity.native: null
+        prospector.scanner.fingerprint.enabled: false
   - id: udp-udp-35051de0-46a5-11ee-8d5d-9f98c8182f60
     name: syslog-udp-514
-    revision: 3
+    revision: 4
     type: udp
     use_output: default
     meta:
       package:
         name: udp
-        version: 1.10.0
+        version:
     data_stream:
       namespace: so
     package_policy_id: 35051de0-46a5-11ee-8d5d-9f98c8182f60
@@ -458,13 +307,13 @@ inputs:
           - syslog
   - id: tcp-tcp-33d37bb0-46a5-11ee-8d5d-9f98c8182f60
     name: syslog-tcp-514
-    revision: 3
+    revision: 4
     type: tcp
     use_output: default
     meta:
       package:
         name: tcp
-        version: 1.10.0
+        version:
     data_stream:
       namespace: so
     package_policy_id: 33d37bb0-46a5-11ee-8d5d-9f98c8182f60

salt/elasticfleet/config.map.jinja

@@ -0,0 +1,34 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+
+{# advanced config_yaml options for elasticfleet logstash output #}
+{% set ADV_OUTPUT_LOGSTASH_RAW = ELASTICFLEETMERGED.config.outputs.logstash %}
+{% set ADV_OUTPUT_LOGSTASH = {} %}
+{% for k, v in ADV_OUTPUT_LOGSTASH_RAW.items() %}
+{%   if v != "" and v is not none %}
+{%     if k == 'queue_mem_events' %}
+{#       rename queue_mem_events queue.mem.events #}
+{%       do ADV_OUTPUT_LOGSTASH.update({'queue.mem.events':v}) %}
+{%     elif k == 'loadbalance' %}
+{%       if v %}
+{#         only include loadbalance config when its True #}
+{%         do ADV_OUTPUT_LOGSTASH.update({k:v}) %}
+{%       endif %}
+{%     else %}
+{%       do ADV_OUTPUT_LOGSTASH.update({k:v}) %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+{% set LOGSTASH_CONFIG_YAML_RAW = [] %}
+{% if ADV_OUTPUT_LOGSTASH %}
+{%   for k, v in ADV_OUTPUT_LOGSTASH.items() %}
+{%     do LOGSTASH_CONFIG_YAML_RAW.append(k ~ ': ' ~ v) %}
+{%   endfor %}
+{% endif %}
+
+{% set LOGSTASH_CONFIG_YAML = LOGSTASH_CONFIG_YAML_RAW | join('\\n') if LOGSTASH_CONFIG_YAML_RAW else '' %}

salt/elasticfleet/config.sls

@@ -11,6 +11,7 @@
 
 include:
   - elasticfleet.artifact_registry
+  - elasticfleet.ssl
 
 # Add EA Group
 elasticfleetgroup:
@@ -95,6 +96,9 @@ soresourcesrepoclone:
     - rev: 'main'
     - depth: 1
     - force_reset: True
+    - retry:
+        attempts: 3
+        interval: 10
 {% endif %}
 
 elasticdefendconfdir:

salt/elasticfleet/defaults.yaml

@@ -10,6 +10,14 @@ elasticfleet:
       grid_enrollment: ''
     defend_filters:
       enable_auto_configuration: False
+    outputs:
+      logstash:
+        bulk_max_size: ''
+        worker: ''
+        queue_mem_events: ''
+        timeout: ''
+        loadbalance: False
+        compression_level: ''
     subscription_integrations: False
     auto_upgrade_integrations: False
   logging:

salt/elasticfleet/enabled.sls

@@ -6,16 +6,17 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKER %}
+{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
 
 {#   This value is generated during node install and stored in minion pillar #}
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
 
 include:
+  - ca
+  - logstash.ssl
   - elasticfleet.config
   - elasticfleet.sostatus
-  - ssl
 
 {% if grains.role not in ['so-fleet'] %}
 # Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready
@@ -36,12 +37,13 @@ so-elastic-fleet-auto-configure-logstash-outputs:
 {# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #}
 so-elastic-fleet-auto-configure-logstash-outputs-force:
   cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-outputs-update --force --certs
+    - name: /usr/sbin/so-elastic-fleet-outputs-update --certs
     - retry:
         attempts: 4
         interval: 30
     - onchanges:
         - x509: etc_elasticfleet_logstash_crt
+        - x509: elasticfleet_kafka_crt
 {% endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection
@@ -92,17 +94,17 @@ so-elastic-fleet:
     - user: 947
     - networks:
       - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-elastic-fleet'].ip }}
+        - ipv4_address: {{ DOCKERMERGED.containers['so-elastic-fleet'].ip }}
     - extra_hosts:
         - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
         - {{ GLOBALS.hostname }}:{{ GLOBALS.node_ip }}
-        {% if DOCKER.containers['so-elastic-fleet'].extra_hosts %}
-          {% for XTRAHOST in DOCKER.containers['so-elastic-fleet'].extra_hosts %}
+        {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_hosts %}
+          {% for XTRAHOST in DOCKERMERGED.containers['so-elastic-fleet'].extra_hosts %}
         - {{ XTRAHOST }}
           {% endfor %}
         {% endif %}
     - port_bindings:
-      {% for BINDING in DOCKER.containers['so-elastic-fleet'].port_bindings %}
+      {% for BINDING in DOCKERMERGED.containers['so-elastic-fleet'].port_bindings %}
       - {{ BINDING }}
       {% endfor %}
     - binds:
@@ -110,8 +112,8 @@ so-elastic-fleet:
       - /etc/pki/elasticfleet-server.key:/etc/pki/elasticfleet-server.key:ro
       - /etc/pki/tls/certs/intca.crt:/etc/pki/tls/certs/intca.crt:ro
       - /opt/so/log/elasticfleet:/usr/share/elastic-agent/logs 
-     {% if DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %}
-        {% for BIND in DOCKER.containers['so-elastic-fleet'].custom_bind_mounts %}
+     {% if DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
+        {% for BIND in DOCKERMERGED.containers['so-elastic-fleet'].custom_bind_mounts %}
       - {{ BIND }}
         {% endfor %}
       {% endif %}      
@@ -126,12 +128,23 @@ so-elastic-fleet:
       - FLEET_CA=/etc/pki/tls/certs/intca.crt     
       - FLEET_SERVER_ELASTICSEARCH_CA=/etc/pki/tls/certs/intca.crt
       - LOGS_PATH=logs
-      {% if DOCKER.containers['so-elastic-fleet'].extra_env %}
-        {% for XTRAENV in DOCKER.containers['so-elastic-fleet'].extra_env %}
+      {% if DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
+        {% for XTRAENV in DOCKERMERGED.containers['so-elastic-fleet'].extra_env %}
       - {{ XTRAENV }}
         {% endfor %}
       {% endif %}
+    {% if DOCKERMERGED.containers['so-elastic-fleet'].ulimits %}
+    - ulimits:
+    {%   for ULIMIT in DOCKERMERGED.containers['so-elastic-fleet'].ulimits %}
+      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
+    {%   endfor %}
+    {% endif %}
     - watch:
+      - file: trusttheca
+      - x509: etc_elasticfleet_key
+      - x509: etc_elasticfleet_crt
+    - require:
+      - file: trusttheca
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
 {%   endif %}

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json

@@ -2,7 +2,7 @@
 {%- raw -%}
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "import-zeek-logs",
@@ -10,19 +10,31 @@
   "description": "Zeek Import logs",
   "policy_id": "so-grid-nodes_general",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/zeek/logs/*.log"
             ],
             "data_stream.dataset": "import",
-            "tags": [],
+            "pipeline": "",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
+            "include_files": [],
             "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"import.file\").slice(0,-4);\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n      imported: true\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
-            "custom": "exclude_files: [\"{%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}.log$\"]\n"
+            "tags": [],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json

@@ -11,36 +11,51 @@
 {%- endif -%}
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "kratos-logs",
-  "namespace": "so",
   "description": "Kratos logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/kratos/kratos.log"
             ],
             "data_stream.dataset": "kratos",
-            "tags": ["so-kratos"],
+            "pipeline": "kratos",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             {%- if valid_identities -%}
             "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos\n- if:\n    has_fields:\n      - identity_id\n  then:{% for id, email in identities %}\n    - if:\n        equals:\n          identity_id: \"{{ id }}\"\n      then:\n        - add_fields:\n            target: ''\n            fields:\n              user.name: \"{{ email }}\"{% endfor %}",
             {%- else -%}
             "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos",
             {%- endif -%}
-            "custom": "pipeline: kratos"
+            "tags": [
+              "so-kratos"
+            ],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
-
+}

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json

@@ -2,28 +2,38 @@
 {%- raw -%}
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
-  "id": "zeek-logs",
   "name": "zeek-logs",
   "namespace": "so",
   "description": "Zeek logs",
   "policy_id": "so-grid-nodes_general",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/zeek/logs/current/*.log"
             ],
             "data_stream.dataset": "zeek",
-            "tags": [],
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
+            "include_files": [],
             "processors": "- dissect:\n    tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n    field: \"log.file.path\"\n    trim_chars: \".log\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\");\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
-            "custom": "exclude_files: [\"{%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}.log$\"]\n"
+            "tags": [],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
@@ -31,4 +41,4 @@
   },
   "force": true
 }
-{%- endraw -%}
+{%- endraw -%}

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -5,7 +5,7 @@
   "package": {
     "name": "endpoint",
     "title": "Elastic Defend",
-    "version": "8.18.1",
+    "version": "9.0.2",
     "requires_root": true
   },
   "enabled": true,

salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json

@@ -1,26 +1,43 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "hydra-logs",
-  "namespace": "so",
   "description": "Hydra logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/hydra/hydra.log"
             ],
             "data_stream.dataset": "hydra",
-            "tags": ["so-hydra"],
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: hydra",
-            "custom": "pipeline: hydra"
+            "pipeline": "hydra",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: hydra",
+            "tags": [
+              "so-hydra"
+            ],
+            "recursive_glob": true,
+            "ignore_older": "72h",
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
@@ -28,3 +45,5 @@
   },
   "force": true
 }
+
+

salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json

@@ -1,30 +1,44 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "idh-logs",
-  "namespace": "so",
   "description": "IDH integration",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/idh/opencanary.log"
             ],
             "data_stream.dataset": "idh",
-            "tags": [],
+            "pipeline": "common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- convert:\n    fields:\n      - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n- drop_fields:\n    when:\n      equals:\n        event.code: \"1001\"\n    fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n    ignore_missing: true\n- rename:\n    fields:\n      - from: \"src_host\"\n        to: \"source.ip\"\n      - from: \"src_port\"\n        to: \"source.port\"\n      - from: \"dst_host\"\n        to: \"destination.host\"\n      - from: \"dst_port\"\n        to: \"destination.port\"\n    ignore_missing: true\n- drop_fields:\n    fields: '[\"prospector\", \"input\", \"offset\", \"beat\"]'\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: opencanary",
-            "custom": "pipeline: common"
+            "tags": [],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -1,33 +1,46 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "import-evtx-logs",
-  "namespace": "so",
   "description": "Import Windows EVTX logs",
   "policy_id": "so-grid-nodes_general",
-  "vars": {},
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/evtx/*.json"
             ],
             "data_stream.dataset": "import",
-            "custom": "",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.6.1\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.6.1\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.6.1\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
-            ]
+            ],
+            "recursive_glob": true,
+            "ignore_older": "72h",
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json

@@ -1,30 +1,45 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "import-suricata-logs",
-  "namespace": "so",
   "description": "Import Suricata logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/suricata/eve*.json"
             ],
             "data_stream.dataset": "import",
+            "pipeline": "suricata.common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
+            "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata\n      imported: true\n- dissect:\n      tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n      field: \"log.file.path\"\n      target_prefix: \"\"\n",
             "tags": [],
-            "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata\n      imported: true\n- dissect:\n      tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n      field: \"log.file.path\"\n      target_prefix: \"\"",
-            "custom": "pipeline: suricata.common"
+            "recursive_glob": true,
+            "ignore_older": "72h",
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json

@@ -15,7 +15,7 @@
           "enabled": true,
           "vars": {
             "paths": [
-              "/opt/so/log/redis/redis.log"
+              "/opt/so/log/redis/redis-server.log"
             ],
             "tags": [
               "redis-log"

salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json

@@ -1,18 +1,17 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "rita-logs",
-  "namespace": "so",
   "description": "RITA Logs",
   "policy_id": "so-grid-nodes_general",
-  "vars": {},
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
@@ -20,15 +19,28 @@
               "/nsm/rita/exploded-dns.csv",
               "/nsm/rita/long-connections.csv"
             ],
-            "exclude_files": [],
-            "ignore_older": "72h",
             "data_stream.dataset": "rita",
-            "tags": [],
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- dissect:\n    tokenizer: \"/nsm/rita/%{pipeline}.csv\"\n    field: \"log.file.path\"\n    trim_chars: \".csv\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\").split(\"-\");\n          if (pl.length > 1) {\n            pl = pl[1];\n          }\n          else {\n            pl = pl[0];\n          }\n          event.Put(\"@metadata.pipeline\", \"rita.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: rita",
-            "custom": "exclude_lines: ['^Score', '^Source', '^Domain', '^No results']"
+            "tags": [],
+            "recursive_glob": true,
+            "ignore_older": "72h",
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json

@@ -1,29 +1,41 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "so-ip-mappings",
-  "namespace": "so",
   "description": "IP Description mappings",
   "policy_id": "so-grid-nodes_general",
-  "vars": {},
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/custom-mappings/ip-descriptions.csv"
             ],
             "data_stream.dataset": "hostnamemappings",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
+            "processors": "- decode_csv_fields:\n    fields:\n      message: decoded.csv\n    separator: \",\"\n    ignore_missing: false\n    overwrite_keys: true\n    trim_leading_space: true\n    fail_on_error: true\n\n- extract_array:\n    field: decoded.csv\n    mappings:\n      so.ip_address: '0'\n      so.description: '1'\n\n- script:\n    lang: javascript\n    source: >\n      function process(event) {\n          var ip = event.Get('so.ip_address');\n          var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n          if (!validIpRegex.test(ip)) {\n              event.Cancel();\n          }\n      }\n- fingerprint:\n    fields: [\"so.ip_address\"]\n    target_field: \"@metadata._id\"\n",
             "tags": [
               "so-ip-mappings"
             ],
-            "processors": "- decode_csv_fields:\n    fields:\n      message: decoded.csv\n    separator: \",\"\n    ignore_missing: false\n    overwrite_keys: true\n    trim_leading_space: true\n    fail_on_error: true\n\n- extract_array:\n    field: decoded.csv\n    mappings:\n      so.ip_address: '0'\n      so.description: '1'\n\n- script:\n    lang: javascript\n    source: >\n      function process(event) {\n          var ip = event.Get('so.ip_address');\n          var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n          if (!validIpRegex.test(ip)) {\n              event.Cancel();\n          }\n      }\n- fingerprint:\n    fields: [\"so.ip_address\"]\n    target_field: \"@metadata._id\"\n",
-            "custom": ""
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
@@ -31,5 +43,3 @@
   },
   "force": true
 }
-
-

salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json

@@ -1,30 +1,44 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "soc-auth-sync-logs",
-  "namespace": "so",
   "description": "Security Onion - Elastic Auth Sync - Logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sync.log"
             ],
             "data_stream.dataset": "soc",
-            "tags": ["so-soc"],
+            "pipeline": "common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- dissect:\n    tokenizer: \"%{event.action}\"\n    field: \"message\"\n    target_prefix: \"\"\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: auth_sync",
-            "custom": "pipeline: common"
+            "tags": [],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json

@@ -1,35 +1,48 @@
 {
-  "policy_id": "so-grid-nodes_general",
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "soc-detections-logs",
   "description": "Security Onion Console - Detections Logs",
+  "policy_id": "so-grid-nodes_general",
   "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/detections_runtime-status_sigma.log",
               "/opt/so/log/soc/detections_runtime-status_yara.log"
             ],
-            "exclude_files": [],
-            "ignore_older": "72h",
             "data_stream.dataset": "soc",
+            "pipeline": "common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: detections\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true",
             "tags": [
               "so-soc"
             ],
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: detections\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true",
-            "custom": "pipeline: common"
+            "recursive_glob": true,
+            "ignore_older": "72h",
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json

@@ -1,30 +1,46 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "soc-salt-relay-logs",
-  "namespace": "so",
   "description": "Security Onion - Salt Relay - Logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/salt-relay.log"
             ],
             "data_stream.dataset": "soc",
-            "tags": ["so-soc"],
+            "pipeline": "common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- dissect:\n    tokenizer: \"%{soc.ts} | %{event.action}\"\n    field: \"message\"\n    target_prefix: \"\"\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: salt_relay",
-            "custom": "pipeline: common"
+            "tags": [
+              "so-soc"
+            ],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json

@@ -1,30 +1,44 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "soc-sensoroni-logs",
-  "namespace": "so",
   "description": "Security Onion - Sensoroni - Logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/sensoroni/sensoroni.log"
             ],
             "data_stream.dataset": "soc",
-            "tags": [],
+            "pipeline": "common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"sensoroni\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: sensoroni\n- rename:\n    fields:\n      - from: \"sensoroni.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"sensoroni.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"sensoroni.fields.method\"\n        to: \"http.request.method\"\n      - from: \"sensoroni.fields.path\"\n        to: \"url.path\"\n      - from: \"sensoroni.message\"\n        to: \"event.action\"\n      - from: \"sensoroni.level\"\n        to: \"log.level\"\n    ignore_missing: true",
-            "custom": "pipeline: common"
+            "tags": [],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
-  "force": true
-}
+"force": true
+}

salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json

@@ -1,30 +1,46 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "soc-server-logs",
-  "namespace": "so",
   "description": "Security Onion Console Logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sensoroni-server.log"
             ],
             "data_stream.dataset": "soc",
-            "tags": ["so-soc"],
+            "pipeline": "common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: server\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true",
-            "custom": "pipeline: common"
+            "tags": [
+              "so-soc"
+            ],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json

@@ -1,30 +1,44 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "strelka-logs",
-  "namespace": "so",
-  "description": "Strelka logs",
+  "description": "Strelka Logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/strelka/log/strelka.log"
             ],
             "data_stream.dataset": "strelka",
-            "tags": [],
+            "pipeline": "strelka.file",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- add_fields:\n    target: event\n    fields:\n      category: file\n      module: strelka",
-            "custom": "pipeline: strelka.file"
+            "tags": [],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json

@@ -1,26 +1,40 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "suricata-logs",
-  "namespace": "so",
   "description": "Suricata integration",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/suricata/eve*.json"
             ],
             "data_stream.dataset": "suricata",
-            "tags": [],
+            "pipeline": "suricata.common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata",
-            "custom": "pipeline: suricata.common"
+            "tags": [],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-grid-nodes.json

@@ -0,0 +1,107 @@
+{
+  "package": {
+    "name": "elasticsearch",
+    "version": ""
+  },
+  "name": "elasticsearch-grid-nodes_heavy",
+  "namespace": "default",
+  "description": "Elasticsearch Logs",
+  "policy_id": "so-grid-nodes_heavy",
+  "inputs": {
+    "elasticsearch-logfile": {
+      "enabled": true,
+      "streams": {
+        "elasticsearch.audit": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/*_audit.json"
+            ]
+          }
+        },
+        "elasticsearch.deprecation": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/*_deprecation.json"
+            ]
+          }
+        },
+        "elasticsearch.gc": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/gc.log.[0-9]*",
+              "/var/log/elasticsearch/gc.log"
+            ]
+          }
+        },
+        "elasticsearch.server": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/elasticsearch/*.json"
+            ]
+          }
+        },
+        "elasticsearch.slowlog": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/*_index_search_slowlog.json",
+              "/var/log/elasticsearch/*_index_indexing_slowlog.json"
+            ]
+          }
+        }
+      }
+    },
+    "elasticsearch-elasticsearch/metrics": {
+      "enabled": false,
+      "vars": {
+        "hosts": [
+          "http://localhost:9200"
+        ],
+        "scope": "node"
+      },
+      "streams": {
+        "elasticsearch.stack_monitoring.ccr": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.cluster_stats": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.enrich": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.index": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.index_recovery": {
+          "enabled": false,
+          "vars": {
+            "active.only": true
+          }
+        },
+        "elasticsearch.stack_monitoring.index_summary": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.ml_job": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.node": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.node_stats": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.pending_tasks": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.shard": {
+          "enabled": false
+        }
+      }
+    }
+  },
+  "force": true
+}

salt/elasticfleet/install_agent_grid.sls

@@ -8,7 +8,9 @@
 {% endif %}
 
 {% set AGENT_STATUS = salt['service.available']('elastic-agent') %}
-{% if not AGENT_STATUS  %}
+{% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %}
+
+{% if not AGENT_STATUS or not AGENT_EXISTS %}
 
 pull_agent_installer:
   file.managed:
@@ -19,7 +21,7 @@ pull_agent_installer:
 
 run_installer:
   cmd.run:
-    - name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }}
+    - name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }} -force
     - cwd: /opt/so
     - retry:
         attempts: 3

salt/elasticfleet/integration-defaults.map.jinja

@@ -21,6 +21,7 @@
     'azure_application_insights.app_state': 'azure.app_state',
     'azure_billing.billing': 'azure.billing',
     'azure_functions.metrics': 'azure.function',
+    'azure_ai_foundry.metrics': 'azure.ai_foundry',
     'azure_metrics.compute_vm_scaleset': 'azure.compute_vm_scaleset',
     'azure_metrics.compute_vm': 'azure.compute_vm',
     'azure_metrics.container_instance': 'azure.container_instance',
@@ -121,6 +122,9 @@
                  "phases": {
                    "cold": {
                      "actions": {
+                        "allocate":{
+                          "number_of_replicas": ""
+                        },
                        "set_priority": {"priority": 0}
                      },
                      "min_age": "60d"
@@ -137,12 +141,31 @@
                          "max_age": "30d",
                          "max_primary_shard_size": "50gb"
                        },
+                       "forcemerge":{
+                         "max_num_segments": ""
+                       },
+                       "shrink":{
+                         "max_primary_shard_size": "",
+                         "method": "COUNT",
+                         "number_of_shards": ""
+                       },
                        "set_priority": {"priority": 100}
                       },
                       "min_age": "0ms"
                    },
                    "warm": {
                      "actions": {
+                        "allocate": {
+                          "number_of_replicas": ""
+                        },
+                        "forcemerge": {
+                          "max_num_segments": ""
+                        },
+                        "shrink":{
+                          "max_primary_shard_size": "",
+                          "method": "COUNT",
+                          "number_of_shards": ""
+                        },
                        "set_priority": {"priority": 50}
                      },
                      "min_age": "30d"

salt/elasticfleet/soc_elasticfleet.yaml

@@ -50,6 +50,46 @@ elasticfleet:
       global: True
       forcedType: bool
       helpLink: elastic-fleet.html
+    outputs:
+      logstash:
+        bulk_max_size:
+          description: The maximum number of events to bulk in a single Logstash request.
+          global: True
+          forcedType: int
+          advanced: True
+          helpLink: elastic-fleet.html
+        worker:
+          description: The number of workers per configured host publishing events.
+          global: True
+          forcedType: int
+          advanced: true
+          helpLink: elastic-fleet.html
+        queue_mem_events:
+          title: queued events
+          description: The number of events the queue can store. This value should be evenly divisible by the smaller of 'bulk_max_size' to avoid sending partial batches to the output.
+          global: True
+          forcedType: int
+          advanced: True
+          helpLink: elastic-fleet.html
+        timeout:
+          description: The number of seconds to wait for responses from the Logstash server before timing out. Eg 30s
+          regex: ^[0-9]+s$
+          advanced: True
+          global: True
+          helpLink: elastic-fleet.html
+        loadbalance:
+          description: If true and multiple Logstash hosts are configured, the output plugin load balances published events onto all Logstash hosts. If false, the output plugin sends all events to one host (determined at random) and switches to another host if the selected one becomes unresponsive.
+          forcedType: bool
+          advanced: True
+          global: True
+          helpLink: elastic-fleet.html
+        compression_level:
+          description: The gzip compression level. The compression level must be in the range of 1 (best speed) to 9 (best compression).
+          regex: ^[1-9]$
+          forcedType: int
+          advanced: True
+          global: True
+          helpLink: elastic-fleet.html
     server:
       custom_fqdn:
         description: Custom FQDN for Agents to connect to. One per line.

salt/elasticfleet/ssl.sls

@@ -0,0 +1,186 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%   from 'ca/map.jinja' import CA %}
+
+{% if GLOBALS.is_manager or GLOBALS.role in ['so-heavynode', 'so-fleet', 'so-receiver'] %}
+
+{% if grains['role'] not in [ 'so-heavynode', 'so-receiver'] %}
+# Start -- Elastic Fleet Host Cert
+etc_elasticfleet_key:
+  x509.private_key_managed:
+    - name: /etc/pki/elasticfleet-server.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/elasticfleet-server.key') -%}
+    - prereq:
+      - x509: etc_elasticfleet_crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+etc_elasticfleet_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/elasticfleet-server.crt
+    - ca_server: {{ CA.server }}
+    - signing_policy: elasticfleet
+    - private_key: /etc/pki/elasticfleet-server.key
+    - CN: {{ GLOBALS.hostname }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %}
+    - days_remaining: 7
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+
+efperms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-server.key
+    - mode: 640
+    - group: 939
+
+chownelasticfleetcrt:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-server.crt
+    - mode: 640
+    - user: 947
+    - group: 939
+
+chownelasticfleetkey:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-server.key
+    - mode: 640
+    - user: 947
+    - group: 939
+# End -- Elastic Fleet Host Cert
+{% endif %} # endif is for not including HeavyNodes & Receivers 
+
+
+# Start -- Elastic Fleet Client Cert for Agent (Mutual Auth with Logstash Output)
+etc_elasticfleet_agent_key:
+  x509.private_key_managed:
+    - name: /etc/pki/elasticfleet-agent.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/elasticfleet-agent.key') -%}
+    - prereq:
+      - x509: etc_elasticfleet_agent_crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+etc_elasticfleet_agent_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/elasticfleet-agent.crt
+    - ca_server: {{ CA.server }}
+    - signing_policy: elasticfleet
+    - private_key: /etc/pki/elasticfleet-agent.key
+    - CN: {{ GLOBALS.hostname }}
+    - days_remaining: 7
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+  cmd.run:
+    - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-agent.key -topk8 -out /etc/pki/elasticfleet-agent.p8 -nocrypt"
+    - onchanges:
+      - x509: etc_elasticfleet_agent_key
+
+efagentperms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-agent.key
+    - mode: 640
+    - group: 939
+
+chownelasticfleetagentcrt:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-agent.crt
+    - mode: 640
+    - user: 947
+    - group: 939
+
+chownelasticfleetagentkey:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-agent.key
+    - mode: 640
+    - user: 947
+    - group: 939
+# End -- Elastic Fleet Client Cert for Agent (Mutual Auth with Logstash Output)
+
+{% endif %}
+
+{% if GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone'] %}
+elasticfleet_kafka_key:
+  x509.private_key_managed:
+    - name: /etc/pki/elasticfleet-kafka.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/elasticfleet-kafka.key') -%}
+    - prereq:
+      - x509: elasticfleet_kafka_crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+elasticfleet_kafka_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/elasticfleet-kafka.crt
+    - ca_server: {{ CA.server }}
+    - signing_policy: kafka
+    - private_key: /etc/pki/elasticfleet-kafka.key
+    - CN: {{ GLOBALS.hostname }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - days_remaining: 7
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+
+elasticfleet_kafka_cert_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-kafka.crt
+    - mode: 640
+    - user: 947
+    - group: 939
+
+elasticfleet_kafka_key_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-kafka.key
+    - mode: 640
+    - user: 947
+    - group: 939
+{% endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -17,9 +17,9 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
 
   # Third, configure Elastic Defend Integration seperately
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
+
   # Initial Endpoints
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json
-  do
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json; do
     printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
     elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
@@ -40,8 +40,7 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   done
 
   # Grid Nodes - General
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json
-  do
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json; do
     printf "\n\nGrid Nodes Policy_General - Loading $INTEGRATION\n"
     elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
@@ -60,13 +59,9 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
       fi
     fi
   done
-  if [[ "$RETURN_CODE" != "1" ]]; then
-    touch /opt/so/state/eaintegrations.txt
-  fi
 
   # Grid Nodes - Heavy
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json
-  do
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json; do
     printf "\n\nGrid Nodes Policy_Heavy - Loading $INTEGRATION\n"
     elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
@@ -78,22 +73,16 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
       fi
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
-      if [ "$NAME" != "elasticsearch-logs" ]; then
-        if ! elastic_fleet_integration_create "@$INTEGRATION"; then
-          echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
-          RETURN_CODE=1
-          continue
-        fi
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
       fi
     fi
   done
-  if [[ "$RETURN_CODE" != "1" ]]; then
-    touch /opt/so/state/eaintegrations.txt
-  fi
 
   # Fleet Server - Optional integrations
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json
-  do
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json; do
     if ! [ "$INTEGRATION" == "/opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json" ]; then
       FLEET_POLICY=`echo "$INTEGRATION"| cut -d'/' -f7`
       printf "\n\nFleet Server Policy - Loading $INTEGRATION\n"
@@ -117,6 +106,8 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
       fi
     fi
   done
+
+  # Only create the state file if all policies were created/updated successfully
   if [[ "$RETURN_CODE" != "1" ]]; then
     touch /opt/so/state/eaintegrations.txt
   fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade

@@ -14,7 +14,7 @@ if ! is_manager_node; then
 fi
 
 # Get current list of Grid Node Agents that need to be upgraded
-RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%20:%20%22{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%22%20and%20policy_id%20:%20%22so-grid-nodes_general%22&showInactive=false&getStatusSummary=true")
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%3A%20{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%20AND%20policy_id%3A%20so-grid-nodes_%2A&showInactive=false&getStatusSummary=true" --retry 3 --retry-delay 30 --fail 2>/dev/null)
 
 # Check to make sure that the server responded with good data - else, bail from script
 CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON")

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update

@@ -26,7 +26,7 @@ function update_es_urls() {
 }
 
 # Get current list of Fleet Elasticsearch URLs
-RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_elasticsearch')
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_elasticsearch' --retry 3 --retry-delay 30 --fail 2>/dev/null)
 
 # Check to make sure that the server responded with good data - else, bail from script
 CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -86,7 +86,7 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         latest_package_list=$(/usr/sbin/so-elastic-fleet-package-list)
         echo '{ "packages" : []}' > $BULK_INSTALL_PACKAGE_LIST
         rm -f $INSTALLED_PACKAGE_LIST
-        echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .savedObject.attributes.install_version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
+        echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .installationInfo.version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
 
         while read -r package; do
             # get package details

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -3,13 +3,16 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%- from 'vars/globals.map.jinja' import GLOBALS %}
+{%- from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%- from 'elasticfleet/config.map.jinja' import LOGSTASH_CONFIG_YAML %}
 
 . /usr/sbin/so-common
 
 FORCE_UPDATE=false
 UPDATE_CERTS=false
+LOGSTASH_PILLAR_CONFIG_YAML="{{ LOGSTASH_CONFIG_YAML }}"
+LOGSTASH_PILLAR_STATE_FILE="/opt/so/state/esfleet_logstash_config_pillar"
 
 while [[ $# -gt 0 ]]; do
   case $1 in
@@ -19,6 +22,7 @@ while [[ $# -gt 0 ]]; do
       ;;
     -c| --certs)
       UPDATE_CERTS=true
+      FORCE_UPDATE=true
       shift
       ;;
     *)
@@ -41,38 +45,45 @@ function update_logstash_outputs() {
         LOGSTASHKEY=$(openssl rsa -in  /etc/pki/elasticfleet-logstash.key)
         LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt)
         LOGSTASHCA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
+        # Revert escaped \\n to \n for jq
+        LOGSTASH_PILLAR_CONFIG_YAML=$(printf '%b' "$LOGSTASH_PILLAR_CONFIG_YAML")
+
         if SECRETS=$(echo "$logstash_policy" | jq -er '.item.secrets' 2>/dev/null); then
             if [[ "$UPDATE_CERTS" != "true"  ]]; then
                 # Reuse existing secret
                 JSON_STRING=$(jq -n \
                     --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
                     --argjson SECRETS "$SECRETS" \
                     --argjson SSL_CONFIG "$SSL_CONFIG" \
-                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": $SSL_CONFIG,"secrets": $SECRETS}')
             else
                 # Update certs, creating new secret
                 JSON_STRING=$(jq -n \
                     --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
                     --arg LOGSTASHKEY "$LOGSTASHKEY" \
                     --arg LOGSTASHCRT "$LOGSTASHCRT" \
                     --arg LOGSTASHCA "$LOGSTASHCA" \
-                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": {"certificate": $LOGSTASHCRT,"certificate_authorities":[ $LOGSTASHCA ]},"secrets": {"ssl":{"key": $LOGSTASHKEY }}}')
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": {"certificate": $LOGSTASHCRT,"certificate_authorities":[ $LOGSTASHCA ]},"secrets": {"ssl":{"key": $LOGSTASHKEY }}}')
             fi
         else
             if [[ "$UPDATE_CERTS" != "true" ]]; then
                 # Reuse existing ssl config
                 JSON_STRING=$(jq -n \
                     --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
                     --argjson SSL_CONFIG "$SSL_CONFIG" \
-                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": $SSL_CONFIG}')
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": $SSL_CONFIG}')
             else
                 # Update ssl config
                 JSON_STRING=$(jq -n \
                     --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
                     --arg LOGSTASHKEY "$LOGSTASHKEY" \
                     --arg LOGSTASHCRT "$LOGSTASHCRT" \
                     --arg LOGSTASHCA "$LOGSTASHCA" \
-                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl": {"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}')
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": {"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}')
             fi
         fi
     fi
@@ -84,19 +95,42 @@ function update_kafka_outputs() {
   # Make sure SSL configuration is included in policy updates for Kafka output. SSL is configured in so-elastic-fleet-setup
   if kafka_policy=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" --fail 2>/dev/null); then
     SSL_CONFIG=$(echo "$kafka_policy" | jq -r '.item.ssl')
+    KAFKAKEY=$(openssl rsa -in  /etc/pki/elasticfleet-kafka.key)
+    KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt)
+    KAFKACA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
     if SECRETS=$(echo "$kafka_policy" | jq -er '.item.secrets' 2>/dev/null); then
-      # Update policy when fleet has secrets enabled
-      JSON_STRING=$(jq -n \
-        --arg UPDATEDLIST "$NEW_LIST_JSON" \
-        --argjson SSL_CONFIG "$SSL_CONFIG" \
-        --argjson SECRETS "$SECRETS" \
-        '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+        if [[ "$UPDATE_CERTS" != "true" ]]; then
+            # Update policy when fleet has secrets enabled
+            JSON_STRING=$(jq -n \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
+              --argjson SSL_CONFIG "$SSL_CONFIG" \
+              --argjson SECRETS "$SECRETS" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+        else
+            # Update certs, creating new secret
+            JSON_STRING=$(jq -n \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
+              --arg KAFKAKEY "$KAFKAKEY" \
+              --arg KAFKACRT "$KAFKACRT" \
+              --arg KAFKACA "$KAFKACA" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": {"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"secrets": {"ssl":{"key": $KAFKAKEY }}}')
+        fi
     else
-      # Update policy when fleet has secrets disabled or policy hasn't been force updated
-      JSON_STRING=$(jq -n \
-        --arg UPDATEDLIST "$NEW_LIST_JSON" \
-        --argjson SSL_CONFIG "$SSL_CONFIG" \
-        '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}')
+        if [[ "$UPDATE_CERTS" != "true" ]]; then
+            # Update policy when fleet has secrets disabled or policy hasn't been force updated
+            JSON_STRING=$(jq -n \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
+              --argjson SSL_CONFIG "$SSL_CONFIG" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}')
+        else
+            # Update ssl config
+            JSON_STRING=$(jq -n \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
+              --arg KAFKAKEY "$KAFKAKEY" \
+              --arg KAFKACRT "$KAFKACRT" \
+              --arg KAFKACA "$KAFKACA" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }}')
+        fi
     fi
     # Update Kafka outputs
     curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
@@ -108,7 +142,7 @@ function update_kafka_outputs() {
 
 {% if GLOBALS.pipeline == "KAFKA" %}
   # Get current list of Kafka Outputs
-  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_kafka')
+  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_kafka' --retry 3 --retry-delay 30 --fail 2>/dev/null)
 
   # Check to make sure that the server responded with good data - else, bail from script
   CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
@@ -119,7 +153,7 @@ function update_kafka_outputs() {
 
   # Get the current list of kafka outputs & hash them
   CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
-  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+  CURRENT_HASH=$(sha256sum <<< "$CURRENT_LIST" | awk '{print $1}')
 
   declare -a NEW_LIST=()
 
@@ -134,7 +168,7 @@ function update_kafka_outputs() {
   {# If global pipeline isn't set to KAFKA then assume default of REDIS / logstash #}
 {% else %}
   # Get current list of Logstash Outputs
-  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash')
+  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash' --retry 3 --retry-delay 30 --fail 2>/dev/null)
 
   # Check to make sure that the server responded with good data - else, bail from script
   CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
@@ -142,10 +176,19 @@ function update_kafka_outputs() {
    printf "Failed to query for current Logstash Outputs..."
    exit 1
   fi
+  # logstash adv config - compare pillar to last state file value
+  if [[ -f "$LOGSTASH_PILLAR_STATE_FILE" ]]; then
+    PREVIOUS_LOGSTASH_PILLAR_CONFIG_YAML=$(cat "$LOGSTASH_PILLAR_STATE_FILE")
+    if [[ "$LOGSTASH_PILLAR_CONFIG_YAML" != "$PREVIOUS_LOGSTASH_PILLAR_CONFIG_YAML" ]]; then
+      echo "Logstash pillar config has changed - forcing update"
+      FORCE_UPDATE=true
+    fi
+    echo "$LOGSTASH_PILLAR_CONFIG_YAML" > "$LOGSTASH_PILLAR_STATE_FILE"
+  fi
 
   # Get the current list of Logstash outputs & hash them
   CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
-  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+  CURRENT_HASH=$(sha256sum <<< "$CURRENT_LIST" | awk '{print $1}')
 
   declare -a NEW_LIST=()
 
@@ -194,7 +237,7 @@ function update_kafka_outputs() {
 
 # Sort & hash the new list of Logstash Outputs
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
-NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
+NEW_HASH=$(sha256sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
 
 # Compare the current & new list of outputs - if different, update the Logstash outputs
 if [[ "$NEW_HASH" = "$CURRENT_HASH" ]] && [[ "$FORCE_UPDATE" != "true" ]]; then

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup

@@ -241,9 +241,11 @@ printf '%s\n'\
     "" >> "$global_pillar_file"
 
 # Call Elastic-Fleet Salt State
+printf "\nApplying elasticfleet state"
 salt-call state.apply elasticfleet queue=True
 
 # Generate installers & install Elastic Agent on the node
 so-elastic-agent-gen-installers
+printf "\nApplying elasticfleet.install_agent_grid state"
 salt-call state.apply elasticfleet.install_agent_grid queue=True
 exit 0

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update

@@ -23,7 +23,7 @@ function update_fleet_urls() {
 }
 
 # Get current list of Fleet Server URLs
-RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default')
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' --retry 3 --retry-delay 30 --fail 2>/dev/null)
 
 # Check to make sure that the server responded with good data - else, bail from script
 CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")

salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy

@@ -34,6 +34,11 @@ if [[ "$RETURN_CODE" != "0" ]]; then
   exit 1
 fi
 
+if [[ ! -f /etc/pki/elasticfleet-kafka.crt || ! -f /etc/pki/elasticfleet-kafka.key ]]; then
+  echo -e "\nKafka certificates not found, can't setup Elastic Fleet output policy for Kafka...\n"
+  exit 1
+fi
+
 KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt)
 KAFKAKEY=$(openssl rsa -in  /etc/pki/elasticfleet-kafka.key)
 KAFKACA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
@@ -47,7 +52,7 @@ if ! kafka_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://l
     --arg KAFKACA "$KAFKACA" \
     --arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \
     --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
-      '{"name":"grid-kafka", "id":"so-manager_kafka","type":"kafka","hosts":[ $MANAGER_IP ],"is_default":false,"is_default_monitoring":false,"config_yaml":"","ssl":{"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"proxy_id":null,"client_id":"Elastic","version": $KAFKA_OUTPUT_VERSION ,"compression":"none","auth_type":"ssl","partition":"round_robin","round_robin":{"group_events":10},"topics":[{"topic":"default-securityonion"}],"headers":[{"key":"","value":""}],"timeout":30,"broker_timeout":30,"required_acks":1,"secrets":{"ssl":{"key": $KAFKAKEY }}}'
+      '{"name":"grid-kafka", "id":"so-manager_kafka","type":"kafka","hosts":[ $MANAGER_IP ],"is_default":false,"is_default_monitoring":false,"config_yaml":"","ssl":{"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"proxy_id":null,"client_id":"Elastic","version": $KAFKA_OUTPUT_VERSION ,"compression":"none","auth_type":"ssl","partition":"round_robin","round_robin":{"group_events":10},"topic":"default-securityonion","headers":[{"key":"","value":""}],"timeout":30,"broker_timeout":30,"required_acks":1,"secrets":{"ssl":{"key": $KAFKAKEY }}}'
     )
     if ! response=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --fail 2>/dev/null); then
       echo -e "\nFailed to setup Elastic Fleet output policy for Kafka...\n"
@@ -67,7 +72,7 @@ elif kafka_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://l
     --arg ENABLED_DISABLED "$ENABLED_DISABLED"\
     --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
     --argjson HOSTS "$HOSTS" \
-      '{"name":"grid-kafka","type":"kafka","hosts":$HOSTS,"is_default":$ENABLED_DISABLED,"is_default_monitoring":$ENABLED_DISABLED,"config_yaml":"","ssl":{"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"proxy_id":null,"client_id":"Elastic","version": $KAFKA_OUTPUT_VERSION ,"compression":"none","auth_type":"ssl","partition":"round_robin","round_robin":{"group_events":10},"topics":[{"topic":"default-securityonion"}],"headers":[{"key":"","value":""}],"timeout":30,"broker_timeout":30,"required_acks":1,"secrets":{"ssl":{"key": $KAFKAKEY }}}'
+      '{"name":"grid-kafka","type":"kafka","hosts":$HOSTS,"is_default":$ENABLED_DISABLED,"is_default_monitoring":$ENABLED_DISABLED,"config_yaml":"","ssl":{"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"proxy_id":null,"client_id":"Elastic","version": $KAFKA_OUTPUT_VERSION ,"compression":"none","auth_type":"ssl","partition":"round_robin","round_robin":{"group_events":10},"topic":"default-securityonion","headers":[{"key":"","value":""}],"timeout":30,"broker_timeout":30,"required_acks":1,"secrets":{"ssl":{"key": $KAFKAKEY }}}'
     )
   if ! response=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" --fail 2>/dev/null); then
     echo -e "\nFailed to force update to Elastic Fleet output policy for Kafka...\n"

salt/elasticsearch/ca.sls

@@ -26,14 +26,14 @@ catrustscript:
         GLOBALS: {{ GLOBALS }}
 {%   endif %}
 
-cacertz:
+elasticsearch_cacerts:
   file.managed:
     - name: /opt/so/conf/ca/cacerts
     - source: salt://elasticsearch/cacerts
     - user: 939
     - group: 939
 
-capemz:
+elasticsearch_capems:
   file.managed:
     - name: /opt/so/conf/ca/tls-ca-bundle.pem
     - source: salt://elasticsearch/tls-ca-bundle.pem

salt/elasticsearch/config.map.jinja

@@ -6,8 +6,6 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %}
 
-{% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
-
 {# this is a list of dicts containing hostname:ip for elasticsearch nodes that need to know about each other for cluster #}
 {% set ELASTICSEARCH_SEED_HOSTS = [] %}
 {% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
@@ -36,14 +34,8 @@
       {% endfor %}
     {% endif %}
 {% elif grains.id.split('_') | last == 'searchnode' %}
-  {% if HIGHLANDER %}
-        {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.roles.extend(['ml', 'master', 'transform']) %}
-  {% endif %}
   {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': [GLOBALS.manager]}}) %}
 {% endif %}
-{% if HIGHLANDER %}
-      {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.xpack.ml.update({'enabled': true}) %}
-{% endif %}
 
 {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.node.update({'name': GLOBALS.hostname}) %}
 {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.cluster.update({'name': GLOBALS.hostname}) %}

salt/elasticsearch/config.sls

@@ -5,11 +5,6 @@
 
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
-
-include:
-  - ssl
-  - elasticsearch.ca
-
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 
@@ -103,10 +98,6 @@ esrolesdir:
     - group: 939
     - makedirs: True
 
-eslibdir:
-  file.absent:
-    - name: /opt/so/conf/elasticsearch/lib
-
 esingestdynamicconf:
   file.recurse:
     - name: /opt/so/conf/elasticsearch/ingest
@@ -124,11 +115,6 @@ esingestconf:
     - group: 939
     - show_changes: False
 
-# Remove .fleet_final_pipeline-1 because we are using global@custom now
-so-fleet-final-pipeline-remove:
-  file.absent:
-    - name: /opt/so/conf/elasticsearch/ingest/.fleet_final_pipeline-1
-
 # Auto-generate Elasticsearch ingest node pipelines from pillar
 {% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %}
 es_ingest_conf_{{pipeline}}:

salt/elasticsearch/defaults.yaml

@@ -1,11 +1,13 @@
 elasticsearch:
   enabled: false
-  version: 8.18.8
+  version: 9.0.8
   index_clean: true
   config:
     action:
       destructive_requires_name: true
     cluster:
+      logsdb:
+        enabled: false
       routing:
         allocation:
           disk:
@@ -72,6 +74,8 @@ elasticsearch:
             actions:
               set_priority:
                 priority: 0
+              allocate:
+                number_of_replicas: ""
             min_age: 60d
           delete:
             actions:
@@ -84,11 +88,25 @@ elasticsearch:
                 max_primary_shard_size: 50gb
               set_priority:
                 priority: 100
+              forcemerge:
+                max_num_segments: ""
+              shrink:
+                max_primary_shard_size: ""
+                method: COUNT
+                number_of_shards: ""
             min_age: 0ms
           warm:
             actions:
               set_priority:
                 priority: 50
+              forcemerge:
+                max_num_segments: ""
+              shrink:
+                max_primary_shard_size: ""
+                method: COUNT
+                number_of_shards: ""
+              allocate:
+                number_of_replicas: ""
             min_age: 30d
     so-case:
       index_sorting: false
@@ -245,7 +263,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-detection:
       index_sorting: false
       index_template:
@@ -284,6 +301,19 @@ elasticsearch:
           hot:
             actions: {}
             min_age: 0ms
+    sos-backup:
+      index_sorting: false
+      index_template:
+        composed_of: []
+        ignore_missing_component_templates: []
+        index_patterns:
+        - sos-backup-*
+        priority: 501
+        template:
+          settings:
+            index:
+              number_of_replicas: 0
+              number_of_shards: 1
     so-assistant-chat:
       index_sorting: false
       index_template:
@@ -584,7 +614,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-import:
       index_sorting: false
       index_template:
@@ -664,7 +693,6 @@ elasticsearch:
                 match_mapping_type: string
           settings:
             index:
-              final_pipeline: .fleet_final_pipeline-1
               lifecycle:
                 name: so-import-logs
               mapping:
@@ -830,53 +858,13 @@ elasticsearch:
         composed_of:
         - agent-mappings
         - dtc-agent-mappings
-        - base-mappings
-        - dtc-base-mappings
-        - client-mappings
-        - dtc-client-mappings
-        - container-mappings
-        - destination-mappings
-        - dtc-destination-mappings
-        - pb-override-destination-mappings
-        - dll-mappings
-        - dns-mappings
-        - dtc-dns-mappings
-        - ecs-mappings
-        - dtc-ecs-mappings
-        - error-mappings
         - event-mappings
-        - dtc-event-mappings
         - file-mappings
-        - dtc-file-mappings
-        - group-mappings
         - host-mappings
         - dtc-host-mappings
         - http-mappings
         - dtc-http-mappings
-        - log-mappings
         - metadata-mappings
-        - network-mappings
-        - dtc-network-mappings
-        - observer-mappings
-        - dtc-observer-mappings
-        - organization-mappings
-        - package-mappings
-        - process-mappings
-        - dtc-process-mappings
-        - related-mappings
-        - rule-mappings
-        - dtc-rule-mappings
-        - server-mappings
-        - service-mappings
-        - dtc-service-mappings
-        - source-mappings
-        - dtc-source-mappings
-        - pb-override-source-mappings
-        - threat-mappings
-        - tls-mappings
-        - url-mappings
-        - user_agent-mappings
-        - dtc-user_agent-mappings
         - common-settings
         - common-dynamic-mappings
         data_stream:
@@ -932,7 +920,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-hydra:
       close: 30
       delete: 365
@@ -1043,7 +1030,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-lists:
       index_sorting: false
       index_template:
@@ -1127,6 +1113,8 @@ elasticsearch:
             actions:
               set_priority:
                 priority: 0
+              allocate:
+                number_of_replicas: ""
             min_age: 60d
           delete:
             actions:
@@ -1139,11 +1127,25 @@ elasticsearch:
                 max_primary_shard_size: 50gb
               set_priority:
                 priority: 100
+              forcemerge:
+                max_num_segments: ""
+              shrink:
+                max_primary_shard_size: ""
+                method: COUNT
+                number_of_shards: ""
             min_age: 0ms
           warm:
             actions:
               set_priority:
                 priority: 50
+              allocate:
+                number_of_replicas: ""
+              forcemerge:
+                max_num_segments: ""
+              shrink:
+                max_primary_shard_size: ""
+                method: COUNT
+                number_of_shards: ""
             min_age: 30d
     so-logs-detections_x_alerts:
       index_sorting: false
@@ -3123,7 +3125,6 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
-      warm: 7
     so-logs-system_x_application:
       index_sorting: false
       index_template:

salt/elasticsearch/enabled.sls

@@ -6,7 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   from 'docker/docker.map.jinja' import DOCKER %}
+{%   from 'docker/docker.map.jinja' import DOCKERMERGED %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
@@ -14,6 +14,9 @@
 {%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
 
 include:
+  - ca
+  - elasticsearch.ca
+  - elasticsearch.ssl
   - elasticsearch.config
   - elasticsearch.sostatus
 
@@ -25,15 +28,15 @@ so-elasticsearch:
     - user: elasticsearch
     - networks:
       - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }}
+        - ipv4_address: {{ DOCKERMERGED.containers['so-elasticsearch'].ip }}
     - extra_hosts:
     {% for node in ELASTICSEARCH_NODES %}
     {%   for hostname, ip in node.items() %}
       - {{hostname}}:{{ip}}
     {%   endfor %}
     {% endfor %}
-    {% if DOCKER.containers['so-elasticsearch'].extra_hosts %}
-      {% for XTRAHOST in DOCKER.containers['so-elasticsearch'].extra_hosts %}
+    {% if DOCKERMERGED.containers['so-elasticsearch'].extra_hosts %}
+      {% for XTRAHOST in DOCKERMERGED.containers['so-elasticsearch'].extra_hosts %}
       - {{ XTRAHOST }}
       {% endfor %}
     {% endif %}
@@ -42,17 +45,19 @@ so-elasticsearch:
       - discovery.type=single-node
       {% endif %}
       - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true
-      ulimits:
-      - memlock=-1:-1
-      - nofile=65536:65536
-      - nproc=4096
-      {% if DOCKER.containers['so-elasticsearch'].extra_env %}
-        {% for XTRAENV in DOCKER.containers['so-elasticsearch'].extra_env %}
+      {% if DOCKERMERGED.containers['so-elasticsearch'].extra_env %}
+        {% for XTRAENV in DOCKERMERGED.containers['so-elasticsearch'].extra_env %}
       - {{ XTRAENV }}
         {% endfor %}
       {% endif %}
+    {% if DOCKERMERGED.containers['so-elasticsearch'].ulimits %}
+    - ulimits:
+    {%   for ULIMIT in DOCKERMERGED.containers['so-elasticsearch'].ulimits %}
+      - {{ ULIMIT.name }}={{ ULIMIT.soft }}:{{ ULIMIT.hard }}
+    {%   endfor %}
+    {% endif %}
     - port_bindings:
-      {% for BINDING in DOCKER.containers['so-elasticsearch'].port_bindings %}
+      {% for BINDING in DOCKERMERGED.containers['so-elasticsearch'].port_bindings %}
       - {{ BINDING }}
       {% endfor %}
     - binds:
@@ -61,11 +66,7 @@ so-elasticsearch:
       - /nsm/elasticsearch:/usr/share/elasticsearch/data:rw
       - /opt/so/log/elasticsearch:/var/log/elasticsearch:rw
       - /opt/so/conf/ca/cacerts:/usr/share/elasticsearch/jdk/lib/security/cacerts:ro
-      {% if GLOBALS.is_manager %}
-      - /etc/pki/ca.crt:/usr/share/elasticsearch/config/ca.crt:ro
-      {% else %}
       - /etc/pki/tls/certs/intca.crt:/usr/share/elasticsearch/config/ca.crt:ro
-      {% endif %}
       - /etc/pki/elasticsearch.crt:/usr/share/elasticsearch/config/elasticsearch.crt:ro
       - /etc/pki/elasticsearch.key:/usr/share/elasticsearch/config/elasticsearch.key:ro
       - /etc/pki/elasticsearch.p12:/usr/share/elasticsearch/config/elasticsearch.p12:ro
@@ -76,28 +77,27 @@ so-elasticsearch:
       - {{ repo }}:{{ repo }}:rw
         {% endfor %}
       {% endif %}
-      {% if DOCKER.containers['so-elasticsearch'].custom_bind_mounts %}
-        {% for BIND in DOCKER.containers['so-elasticsearch'].custom_bind_mounts %}
+      {% if DOCKERMERGED.containers['so-elasticsearch'].custom_bind_mounts %}
+        {% for BIND in DOCKERMERGED.containers['so-elasticsearch'].custom_bind_mounts %}
       - {{ BIND }}
         {% endfor %}
       {% endif %}
     - watch:
-      - file: cacertz
+      - file: trusttheca
+      - x509: elasticsearch_crt
+      - x509: elasticsearch_key
+      - file: elasticsearch_cacerts
       - file: esyml
     - require:
+      - file: trusttheca
+      - x509: elasticsearch_crt
+      - x509: elasticsearch_key
+      - file: elasticsearch_cacerts
       - file: esyml
       - file: eslog4jfile
       - file: nsmesdir
       - file: eslogdir
-      - file: cacertz
-      - x509: /etc/pki/elasticsearch.crt
-      - x509: /etc/pki/elasticsearch.key
       - file: elasticp12perms
-      {% if GLOBALS.is_manager %}
-      - x509: pki_public_ca_crt
-      {% else %}
-      - x509: trusttheca
-      {% endif %}
       - cmd: auth_users_roles_inode
       - cmd: auth_users_inode
 

salt/elasticsearch/files/ingest/common

@@ -1,5 +1,3 @@
-{%- set HIGHLANDER = salt['pillar.get']('global:highlander', False) -%}
-{%- raw -%}
 {
   "description" : "common",
   "processors" : [
@@ -67,19 +65,7 @@
     { "append":          { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  } },
     { "grok":            { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} },
     { "set":             { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
-    { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
-{%- endraw %}
-{%- if HIGHLANDER %}
-    ,
-    {
-      "pipeline": {
-        "name": "ecs"
-      }
-    }
-{%- endif %}
-{%- raw %}
-    ,
+    { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } },
     { "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } }
   ]
 }
-{% endraw %}

salt/elasticsearch/files/ingest/global@custom

@@ -1,31 +1,212 @@
 {
-  "version": 3,
-  "_meta": {
-    "managed_by": "securityonion",
-    "managed": true
-  },
-  "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n",
-  "processors": [
-    { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
-    { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
-    { "split":      { "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')", "field":"data_stream.dataset", "separator":"\\.", "target_field":"datastream_dataset_temp", "ignore_missing":true } },
-    { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
-    { "set":        { "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'", "field":"event.module", "value":"{{ datastream_dataset_temp.0 }}", "ignore_failure":true, "ignore_empty_value":true, "description":"Fix EA network packet capture" } },
-    { "gsub":       { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
-    { "append":     { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}", "allow_duplicates": false  } },
-    { "set":        { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
-    { "set":        { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
-    { "set":        { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
-    { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
-    { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.dataset", "value": "import" } },
-    { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.namespace", "value": "so" } },
-    { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
-    { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
-    { "rename":       { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
-    { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
-    { "set":        { "if": "ctx.event?.dataset != null && ctx.event?.dataset == 'elasticsearch.server'", "field": "event.module", "value":"elasticsearch" }},
-    {"append":      {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null","ignore_failure":true}},
-    {"foreach":     {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"if":"ctx?.event?.module == 'endpoint' && ctx?.host?.ip != null","ignore_missing":true, "description":"Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"}},
-    { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
-  ]
-}
+    "version": 3,
+    "_meta": {
+        "managed_by": "securityonion",
+        "managed": true
+    },
+    "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n",
+    "processors": [
+        {
+            "set": {
+                "ignore_failure": true,
+                "field": "event.module",
+                "value": "elastic_agent"
+            }
+        },
+        {
+            "split": {
+                "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')",
+                "field": "event.dataset",
+                "separator": "\\.",
+                "target_field": "module_temp"
+            }
+        },
+        {
+            "split": {
+                "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')",
+                "field": "data_stream.dataset",
+                "separator": "\\.",
+                "target_field": "datastream_dataset_temp",
+                "ignore_missing": true
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.module_temp != null",
+                "override": true,
+                "field": "event.module",
+                "value": "{{module_temp.0}}"
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'",
+                "field": "event.module",
+                "value": "{{ datastream_dataset_temp.0 }}",
+                "ignore_failure": true,
+                "ignore_empty_value": true,
+                "description": "Fix EA network packet capture"
+            }
+        },
+        {
+            "gsub": {
+                "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')",
+                "field": "event.dataset",
+                "pattern": "^[^.]*.",
+                "replacement": "",
+                "target_field": "dataset_tag_temp"
+            }
+        },
+        {
+            "append": {
+                "if": "ctx.dataset_tag_temp != null",
+                "field": "tags",
+                "value": "{{dataset_tag_temp}}",
+                "allow_duplicates": false
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.network?.direction == 'egress'",
+                "override": true,
+                "field": "network.initiated",
+                "value": "true"
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.network?.direction == 'ingress'",
+                "override": true,
+                "field": "network.initiated",
+                "value": "false"
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.network?.type == 'ipv4'",
+                "override": true,
+                "field": "destination.ipv6",
+                "value": "false"
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.network?.type == 'ipv6'",
+                "override": true,
+                "field": "destination.ipv6",
+                "value": "true"
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.tags != null && ctx.tags.contains('import')",
+                "override": true,
+                "field": "data_stream.dataset",
+                "value": "import"
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.tags != null && ctx.tags.contains('import')",
+                "override": true,
+                "field": "data_stream.namespace",
+                "value": "so"
+            }
+        },
+        {
+            "community_id": {
+                "if": "ctx.event?.dataset == 'endpoint.events.network'",
+                "ignore_failure": true
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.event?.module == 'fim'",
+                "override": true,
+                "field": "event.module",
+                "value": "file_integrity"
+            }
+        },
+        {
+            "rename": {
+                "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'",
+                "ignore_missing": true,
+                "field": "winlog.event_data.Threat Name",
+                "target_field": "winlog.event_data.threat_name"
+            }
+        },
+        {
+            "set": {
+                "if": "ctx?.metadata?.kafka != null",
+                "field": "kafka.id",
+                "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}",
+                "ignore_failure": true
+            }
+        },
+        {
+            "set": {
+                "if": "ctx.event?.dataset != null && ctx.event?.dataset == 'elasticsearch.server'",
+                "field": "event.module",
+                "value": "elasticsearch"
+            }
+        },
+        {
+            "append": {
+                "field": "related.ip",
+                "value": [
+                    "{{source.ip}}",
+                    "{{destination.ip}}"
+                ],
+                "allow_duplicates": false,
+                "if": "ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null",
+                "ignore_failure": true
+            }
+        },
+        {
+            "foreach": {
+                "field": "host.ip",
+                "processor": {
+                    "append": {
+                        "field": "related.ip",
+                        "value": "{{_ingest._value}}",
+                        "allow_duplicates": false
+                    }
+                },
+                "if": "ctx?.event?.module == 'endpoint' && ctx?.host?.ip != null",
+                "ignore_missing": true,
+                "description": "Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"
+            }
+        },
+        {
+            "pipeline": {
+                "name": ".fleet_final_pipeline-1",
+                "ignore_missing_pipeline": true
+            }
+        },
+        {
+            "remove": {
+                "field": "event.agent_id_status",
+                "ignore_missing": true,
+                "if": "ctx?.event?.agent_id_status == 'auth_metadata_missing'"
+            }
+        },
+        {
+            "remove": {
+                "field": [
+                    "message2",
+                    "type",
+                    "fields",
+                    "category",
+                    "module",
+                    "dataset",
+                    "event.dataset_temp",
+                    "dataset_tag_temp",
+                    "module_temp",
+                    "datastream_dataset_temp"
+                ],
+                "ignore_missing": true,
+                "ignore_failure": true
+            }
+        }
+    ]
+}

salt/elasticsearch/files/ingest/kratos

@@ -1,9 +1,98 @@
 {
-  "description" : "kratos",
-  "processors" : [
-    {"set":{"field":"audience","value":"access","override":false,"ignore_failure":true}},
-    {"set":{"field":"event.dataset","ignore_empty_value":true,"ignore_failure":true,"value":"kratos.{{{audience}}}","media_type":"text/plain"}},
-    {"set":{"field":"event.action","ignore_failure":true,"copy_from":"msg"    }},
-    { "pipeline":    { "name": "common" } }
-  ]
+    "description": "kratos",
+    "processors": [
+        {
+            "set": {
+                "field": "audience",
+                "value": "access",
+                "override": false,
+                "ignore_failure": true
+            }
+        },
+        {
+            "set": {
+                "field": "event.dataset",
+                "ignore_empty_value": true,
+                "ignore_failure": true,
+                "value": "kratos.{{{audience}}}",
+                "media_type": "text/plain"
+            }
+        },
+        {
+            "set": {
+                "field": "event.action",
+                "ignore_failure": true,
+                "copy_from": "msg"
+            }
+        },
+        {
+            "rename": {
+                "field": "http_request",
+                "target_field": "http.request",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "http_response",
+                "target_field": "http.response",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "http.request.path",
+                "target_field": "http.uri",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "http.request.method",
+                "target_field": "http.method",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "http.request.method",
+                "target_field": "http.method",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "http.request.query",
+                "target_field": "http.query",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "http.request.headers.user-agent",
+                "target_field": "http.useragent",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "rename": {
+                "field": "file",
+                "target_field": "file.path",
+                "ignore_failure": true,
+                "ignore_missing": true
+            }
+        },
+        {
+            "pipeline": {
+                "name": "common"
+            }
+        }
+    ]
 }

salt/elasticsearch/soc_elasticsearch.yaml

@@ -27,6 +27,13 @@ elasticsearch:
         readonly: True
         global: True
         helpLink: elasticsearch.html
+      logsdb:
+        enabled:
+          description: Enables or disables the Elasticsearch logsdb index mode. When enabled, most logs-* datastreams will convert to logsdb from standard after rolling over.
+          forcedType: bool
+          global: True
+          advanced: True
+          helpLink: elasticsearch.html
       routing:
         allocation:
           disk:
@@ -77,13 +84,6 @@ elasticsearch:
     custom008: *pipelines
     custom009: *pipelines
     custom010: *pipelines
-  managed_integrations:
-    description: List of integrations to add into SOC config UI. Enter the full or partial integration name. Eg. 1password, 1pass
-    forcedType: "[]string"
-    multiline: True
-    global: True
-    advanced: True
-    helpLink: elasticsearch.html
   index_settings:
     global_overrides:
       index_template:
@@ -131,6 +131,47 @@ elasticsearch:
                   description: Maximum primary shard size. Once an index reaches this limit, it will be rolled over into a new index.
                   global: True
                   helpLink: elasticsearch.html
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                  forcedType: string
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
           cold:
             min_age:
               description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
@@ -144,6 +185,12 @@ elasticsearch:
                   description: Used for index recovery after a node restart. Indices with higher priorities are recovered before indices with lower priorities.
                   global: True
                   helpLink: elasticsearch.html
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
           warm:
             min_age: 
               description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
@@ -158,6 +205,52 @@ elasticsearch:
                   forcedType: int
                   global: True
                   helpLink: elasticsearch.html
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
           delete:
             min_age:
               description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
@@ -287,6 +380,47 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch.html
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                  forcedType: string
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
           warm:
             min_age:
               description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
@@ -314,6 +448,52 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch.html
+              shrink:
+                method:
+                  description: Shrink the index to a new index with fewer primary shards. Shrink operation is by count or size.
+                  options:
+                  - COUNT
+                  - SIZE
+                  global: True
+                  advanced: True
+                number_of_shards:
+                  title: shard count
+                  description: Desired shard count. Note that this value is only used when the shrink method selected is 'COUNT'.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                max_primary_shard_size:
+                  title: max shard size
+                  description: Desired shard size in gb/tb/pb eg. 100gb. Note that this value is only used when the shrink method selected is 'SIZE'.
+                  regex: ^[0-9]+(?:gb|tb|pb)$
+                  global: True
+                  forcedType: string
+                  advanced: True
+                allow_write_after_shrink:
+                  description: Allow writes after shrink.
+                  global: True
+                  forcedType: bool
+                  default: False
+                  advanced: True
+              forcemerge:
+                max_num_segments:
+                  description: Reduce the number of segments in each index shard and clean up deleted documents.
+                  global: True
+                  forcedType: int
+                  advanced: True
+                index_codec:
+                  title: compression
+                  description: Use higher compression for stored fields at the cost of slower performance.
+                  forcedType: bool
+                  global: True
+                  default: False
+                  advanced: True
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
           cold:
             min_age:
               description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
@@ -330,6 +510,12 @@ elasticsearch:
                   global: True
                   advanced: True
                   helpLink: elasticsearch.html
+              allocate:
+                number_of_replicas:
+                  description: Set the number of replicas. Remains the same as the previous phase by default.
+                  forcedType: int
+                  global: True
+                  advanced: True
           delete:
             min_age:
               description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.

salt/elasticsearch/ssl.sls

@@ -0,0 +1,66 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'ca/map.jinja' import CA %}
+
+# Create a cert for elasticsearch
+elasticsearch_key:
+  x509.private_key_managed:
+    - name: /etc/pki/elasticsearch.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/elasticsearch.key') -%}
+    - prereq:
+      - x509: /etc/pki/elasticsearch.crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+elasticsearch_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/elasticsearch.crt
+    - ca_server: {{ CA.server }}
+    - signing_policy: registry
+    - private_key: /etc/pki/elasticsearch.key
+    - CN: {{ GLOBALS.hostname }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - days_remaining: 7
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+  cmd.run:
+    - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:"
+    - onchanges:
+      - x509: /etc/pki/elasticsearch.key
+
+elastickeyperms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticsearch.key
+    - mode: 640
+    - group: 930
+    
+elasticp12perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticsearch.p12
+    - mode: 640
+    - group: 930
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/elasticsearch/template.map.jinja

@@ -61,5 +61,55 @@
 {%       do settings.index_template.template.settings.index.pop('sort') %}
 {%     endif %}
 {%   endif %}
+
+{# advanced ilm actions #}
+{%   if settings.policy is defined and settings.policy.phases is defined %}
+{%     set PHASE_NAMES = ["hot", "warm", "cold"] %}
+{%     for P in PHASE_NAMES %}
+{%       if settings.policy.phases[P] is defined and settings.policy.phases[P].actions is defined %}
+{%         set PHASE = settings.policy.phases[P].actions %}
+{#         remove allocate action if number_of_replicas isn't configured #}
+{%         if PHASE.allocate is defined %}
+{%           if PHASE.allocate.number_of_replicas is not defined or PHASE.allocate.number_of_replicas == "" %}
+{%             do PHASE.pop('allocate', none) %}
+{%           endif %}
+{%         endif %}
+{#         start shrink action #}
+{%         if PHASE.shrink is defined %}
+{%           if PHASE.shrink.method is defined %}
+{%             if PHASE.shrink.method == 'COUNT' and PHASE.shrink.number_of_shards is defined and PHASE.shrink.number_of_shards %}
+{#               remove max_primary_shard_size value when doing shrink operation by count vs size #}
+{%               do PHASE.shrink.pop('max_primary_shard_size', none) %}
+{%             elif PHASE.shrink.method == 'SIZE' and PHASE.shrink.max_primary_shard_size is defined and PHASE.shrink.max_primary_shard_size %}
+{#               remove number_of_shards value when doing shrink operation by size vs count #}
+{%               do PHASE.shrink.pop('number_of_shards', none) %}
+{%             else %}
+{#               method isn't defined or missing a required config number_of_shards/max_primary_shard_size #}
+{%               do PHASE.pop('shrink', none) %}
+{%             endif %}
+{%           endif %}
+{%         endif %}
+{#         always remove shrink method since its only used for SOC config, not in the actual ilm policy #}
+{%         if PHASE.shrink is defined %}
+{%            do PHASE.shrink.pop('method', none) %}
+{%         endif %}
+{#         end shrink action #}
+{#         start force merge #}
+{%         if PHASE.forcemerge is defined %}
+{%           if PHASE.forcemerge.index_codec is defined and PHASE.forcemerge.index_codec %}
+{%             do PHASE.forcemerge.update({'index_codec': 'best_compression'}) %}
+{%           else %}
+{%             do PHASE.forcemerge.pop('index_codec', none) %}
+{%           endif %}
+{%           if PHASE.forcemerge.max_num_segments is not defined or not PHASE.forcemerge.max_num_segments %}
+{#             max_num_segments is empty, drop it #}
+{%             do PHASE.pop('forcemerge', none) %}
+{%           endif %}
+{%         endif %}
+{#         end force merge #}
+{%       endif %}
+{%     endfor %}
+{%   endif %}
+
 {%   do ES_INDEX_SETTINGS.update({index | replace("_x_", "."): ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index]}) %}
 {% endfor %}

salt/elasticsearch/templates/component/ecs/dns.json

@@ -1,91 +1,103 @@
 {
-  "_meta": {
-    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-dns.html",
-    "ecs_version": "1.12.2"
-  },
-  "template": {
-    "mappings": {
-      "properties": {
-        "dns": {
-          "properties": {
-            "answers": {
-              "properties": {
-                "class": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "data": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "name": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "ttl": {
-                  "type": "long"
-                },
-                "type": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
+    "_meta": {
+        "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-dns.html",
+        "ecs_version": "1.12.2"
+    },
+    "template": {
+        "mappings": {
+            "properties": {
+                "dns": {
+                    "properties": {
+                        "answers": {
+                            "properties": {
+                                "class": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "data": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "name": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "ttl": {
+                                    "type": "long"
+                                },
+                                "type": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                }
+                            },
+                            "type": "object"
+                        },
+                        "header_flags": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "id": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "op_code": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "question": {
+                            "properties": {
+                                "class": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "name": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "registered_domain": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "subdomain": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "top_level_domain": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "type": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                }
+                            }
+                        },
+                        "query": {
+                            "properties" :{
+                                "type":{
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                },
+                                "type_name": {
+                                    "ignore_above": 1024,
+                                    "type": "keyword"
+                                }
+                            }
+                        },
+                        "resolved_ip": {
+                            "type": "ip"
+                        },
+                        "response_code": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        },
+                        "type": {
+                            "ignore_above": 1024,
+                            "type": "keyword"
+                        }
+                    }
                 }
-              },
-              "type": "object"
-            },
-            "header_flags": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "id": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "op_code": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "question": {
-              "properties": {
-                "class": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "name": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "registered_domain": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "subdomain": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "top_level_domain": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "type": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                }
-              }
-            },
-            "resolved_ip": {
-              "type": "ip"
-            },
-            "response_code": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "type": {
-              "ignore_above": 1024,
-              "type": "keyword"
             }
-          }
         }
-      }
     }
-  }
 }

salt/elasticsearch/templates/component/ecs/file.json

@@ -15,6 +15,13 @@
               "ignore_above": 1024,
               "type": "keyword"
             },
+            "bytes": {
+                "properties": {
+                    "missing": {
+                        "type": "long"
+                    }
+                }
+            },
             "code_signature": {
               "properties": {
                 "digest_algorithm": {