pcapout still used for extracts

update 2.4 references to 3

.github/.gitleaks.toml

@@ -541,5 +541,6 @@ paths = [
     '''gitleaks.toml''',
     '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',
     '''(go.mod|go.sum)$''',
-    '''salt/nginx/files/enterprise-attack.json'''
+    '''salt/nginx/files/enterprise-attack.json''',
+    '''(.*?)whl$'''
 ]

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -2,13 +2,11 @@ body:
   - type: markdown
     attributes:
       value: |
-        ⚠️ This category is solely for conversations related to Security Onion 2.4 ⚠️
-
         If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
   - type: dropdown
     attributes:
       label: Version
-      description: Which version of Security Onion 2.4.x are you asking about?
+      description: Which version of Security Onion are you asking about?
       options:
         -
         - 2.4.10
@@ -26,6 +24,16 @@ body:
         - 2.4.120
         - 2.4.130
         - 2.4.140
+        - 2.4.141
+        - 2.4.150
+        - 2.4.160
+        - 2.4.170
+        - 2.4.180
+        - 2.4.190
+        - 2.4.200
+        - 2.4.201
+        - 2.4.210
+        - 3.0.0
         - Other (please provide detail below)
     validations:
       required: true
@@ -87,7 +95,7 @@ body:
     attributes:
       label: Hardware Specs
       description: >
-        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://docs.securityonion.net/en/2.4/hardware.html?
+        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://securityonion.net/docs/hardware?
       options:
         -
         - Meets minimum requirements

.github/workflows/pythontest.yml

@@ -1,14 +1,10 @@
 name: python-test
 
 on:
-  push:
-    paths: 
-      - "salt/sensoroni/files/analyzers/**"
-      - "salt/manager/tools/sbin"
   pull_request:
     paths:
       - "salt/sensoroni/files/analyzers/**"
-      - "salt/manager/tools/sbin"
+      - "salt/manager/tools/sbin/**"
 
 jobs:
   build:
@@ -17,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.13"]
         python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"]
 
     steps:
@@ -36,4 +32,4 @@ jobs:
         flake8 ${{ matrix.python-code-path }} --show-source --max-complexity=12  --doctests --max-line-length=200 --statistics
     - name: Test with pytest
       run: |
-        pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini
+        PYTHONPATH=${{ matrix.python-code-path }} pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini

.gitignore

@@ -1,4 +1,3 @@
-
 # Created by https://www.gitignore.io/api/macos,windows
 # Edit at https://www.gitignore.io/?templates=macos,windows
 
@@ -67,4 +66,4 @@ __pycache__
 
 # Analyzer dev/test config files
 *_dev.yaml
-site-packages
+site-packages

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.141-20250331 ISO image released on 2025/03/31
+### 2.4.210-20260302 ISO image released on 2026/03/02
 
 
 ### Download and Verify
 
-2.4.141-20250331 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.141-20250331.iso
+2.4.210-20260302 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.210-20260302.iso
  
-MD5: CAE347BC0437A93DC8F4089973ED0EA7  
-SHA1: 3A6F0C2F3B6E3625E06F67EB251372D7E592CB0E  
-SHA256: D0426D8E55E01A0FBA15AFE0BB7887CCB724C07FE82DA706CD1592E6001CD12B  
+MD5: 575F316981891EBED2EE4E1F42A1F016  
+SHA1: 600945E8823221CBC5F1C056084A71355308227E  
+SHA256: A6AA6471125F07FA6E2796430E94BEAFDEF728E833E9728FDFA7106351EBC47E  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.141-20250331.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.210-20260302.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.141-20250331.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.210-20260302.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.141-20250331.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.210-20260302.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.141-20250331.iso.sig securityonion-2.4.141-20250331.iso
+gpg --verify securityonion-2.4.210-20260302.iso.sig securityonion-2.4.210-20260302.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Fri 28 Mar 2025 06:28:11 PM EDT using RSA key ID FE507013
+gpg: Signature made Mon 02 Mar 2026 11:55:24 AM EST using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.
@@ -50,4 +50,4 @@ Primary key fingerprint: C804 A93D 36BE 0C73 3EA1  9644 7C10 60B7 FE50 7013
 If it fails to verify, try downloading again. If it still fails to verify, try downloading from another computer or another network.
 
 Once you've verified the ISO image, you're ready to proceed to our Installation guide:  
-https://docs.securityonion.net/en/2.4/installation.html
+https://securityonion.net/docs/installation

README.md

@@ -1,50 +1,58 @@
-## Security Onion 2.4
+<p align="center">
+  <img src="https://securityonionsolutions.com/logo/logo-so-onion-dark.svg" width="400" alt="Security Onion Logo">
+</p>
 
-Security Onion 2.4 is here!
+# Security Onion
 
-## Screenshots
+Security Onion is a free and open Linux distribution for threat hunting, enterprise security monitoring, and log management. It includes a comprehensive suite of tools designed to work together to provide visibility into your network and host activity.
 
-Alerts
-![Alerts](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/50_alerts.png)
+## ✨ Features
 
-Dashboards
-![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_dashboards.png)
+Security Onion includes everything you need to monitor your network and host systems:
 
-Hunt
-![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/56_hunt.png)
+*   **Security Onion Console (SOC)**: A unified web interface for analyzing security events and managing your grid.
+*   **Elastic Stack**: Powerful search backed by Elasticsearch.
+*   **Intrusion Detection**: Network-based IDS with Suricata and host-based monitoring with Elastic Fleet.
+*   **Network Metadata**: Detailed network metadata generated by Zeek or Suricata.
+*   **Full Packet Capture**: Retain and analyze raw network traffic with Suricata PCAP.
 
-Detections
-![Detections](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_detections.png)
+## ⭐ Security Onion Pro
 
-PCAP
-![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/62_pcap.png)
+For organizations and enterprises requiring advanced capabilities, **Security Onion Pro** offers additional features designed for scale and efficiency:
 
-Grid
-![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/75_grid.png)
+*   **Onion AI**: Leverage powerful AI-driven insights to accelerate your analysis and investigations.
+*   **Enterprise Features**: Enhanced tools and integrations tailored for enterprise-grade security operations.
 
-Config
-![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/87_config.png)
+For more information, visit the [Security Onion Pro](https://securityonionsolutions.com/pro) page.
 
-### Release Notes
+## ☁️ Cloud Deployment
 
-https://docs.securityonion.net/en/2.4/release-notes.html
+Security Onion is available and ready to deploy in the **AWS**, **Azure**, and **Google Cloud (GCP)** marketplaces.
 
-### Requirements
+## 🚀 Getting Started
 
-https://docs.securityonion.net/en/2.4/hardware.html
+| Goal | Resource |
+| :--- | :--- |
+| **Download** | [Security Onion ISO](https://securityonion.net/docs/download) |
+| **Requirements** | [Hardware Guide](https://securityonion.net/docs/hardware) |
+| **Install** | [Installation Instructions](https://securityonion.net/docs/installation) |
+| **What's New** | [Release Notes](https://securityonion.net/docs/release-notes) |
 
-### Download
+## 📖 Documentation & Support
 
-https://docs.securityonion.net/en/2.4/download.html
+For more detailed information, please visit our [Documentation](https://docs.securityonion.net).
 
-### Installation
+*   **FAQ**: [Frequently Asked Questions](https://securityonion.net/docs/faq)
+*   **Community**: [Discussions & Support](https://securityonion.net/docs/community-support)
+*   **Training**: [Official Training](https://securityonion.net/training)
 
-https://docs.securityonion.net/en/2.4/installation.html
+## 🤝 Contributing
 
-### FAQ
+We welcome contributions! Please see our [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on how to get involved.
 
-https://docs.securityonion.net/en/2.4/faq.html
+## 🛡️ License
 
-### Feedback
+Security Onion is licensed under the terms of the license found in the [LICENSE](LICENSE) file.
 
-https://docs.securityonion.net/en/2.4/community-support.html
+---
+*Built with 🧅 by Security Onion Solutions.*

SECURITY.md

@@ -4,6 +4,7 @@
 
 | Version | Supported          |
 | ------- | ------------------ |
+| 3.x     | :white_check_mark: |
 | 2.4.x   | :white_check_mark: |
 | 2.3.x   | :x:                |
 | 16.04.x | :x:                |

VERSION

@@ -1 +1 @@
-2.4.141
+3.0.0

pillar/ca/init.sls

@@ -0,0 +1,2 @@
+ca:
+  server:

pillar/hypervisor/nodes.sls

@@ -0,0 +1,34 @@
+{% set node_types = {} %}
+{% for minionid, ip in salt.saltutil.runner(
+    'mine.get',
+    tgt='G@role:so-hypervisor or G@role:so-managerhype',
+    fun='network.ip_addrs',
+    tgt_type='compound') | dictsort()
+%}
+
+# only add a node to the pillar if it returned an ip from the mine
+{%   if ip | length > 0%}
+{%     set hostname = minionid.split('_') | first %}
+{%     set node_type = minionid.split('_') | last %}
+{%     if node_type not in node_types.keys() %}
+{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%     else %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: ip[0]}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update(ip[0]) %}
+{%       endif %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+
+hypervisor:
+  nodes:
+{% for node_type, values in node_types.items() %}
+    {{node_type}}:
+{%   for hostname, ip in values.items() %}
+      {{hostname}}:
+        ip: {{ip}}
+{%   endfor %}
+{% endfor %}

pillar/node_data/ips.sls

@@ -24,6 +24,7 @@
 {%   endif %}
 {% endfor %}
 
+{% if node_types %}
 node_data:
 {% for node_type, host_values in node_types.items() %}
 {%   for hostname, details in host_values.items() %}
@@ -33,3 +34,6 @@ node_data:
     role: {{node_type}}
 {%   endfor %}
 {% endfor %}
+{% else %}
+node_data: False
+{% endif %}

pillar/top.sls

@@ -1,5 +1,6 @@
 base:
   '*':
+    - ca
     - global.soc_global
     - global.adv_global
     - docker.soc_docker
@@ -18,16 +19,22 @@ base:
     - telegraf.adv_telegraf
     - versionlock.soc_versionlock
     - versionlock.adv_versionlock
+    - soc.license
 
   '* and not *_desktop':
     - firewall.soc_firewall
     - firewall.adv_firewall
     - nginx.soc_nginx
     - nginx.adv_nginx
-    - node_data.ips
 
-  '*_manager or *_managersearch':
+  'salt-cloud:driver:libvirt':
+    - match: grain
+    - vm.soc_vm
+    - vm.adv_vm
+
+  '*_manager or *_managersearch or *_managerhype':
     - match: compound
+    - node_data.ips
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
@@ -37,14 +44,11 @@ base:
     - secrets
     - manager.soc_manager
     - manager.adv_manager
-    - idstools.soc_idstools
-    - idstools.adv_idstools
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
     - soc.soc_soc
     - soc.adv_soc
-    - soc.license
     - kibana.soc_kibana
     - kibana.adv_kibana
     - kratos.soc_kratos
@@ -70,6 +74,9 @@ base:
     - kafka.nodes
     - kafka.soc_kafka
     - kafka.adv_kafka
+    - hypervisor.nodes
+    - hypervisor.soc_hypervisor
+    - hypervisor.adv_hypervisor
     - stig.soc_stig
 
   '*_sensor':
@@ -80,16 +87,14 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
-    - pcap.soc_pcap
-    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
-    - soc.license
 
   '*_eval':
+    - node_data.ips
     - secrets
     - healthcheck.eval
     - elasticsearch.index_templates
@@ -109,11 +114,8 @@ base:
     - elastalert.adv_elastalert
     - manager.soc_manager
     - manager.adv_manager
-    - idstools.soc_idstools
-    - idstools.adv_idstools
     - soc.soc_soc
     - soc.adv_soc
-    - soc.license
     - kibana.soc_kibana
     - kibana.adv_kibana
     - strelka.soc_strelka
@@ -130,14 +132,13 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
-    - pcap.soc_pcap
-    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
   '*_standalone':
+    - node_data.ips
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
@@ -150,8 +151,6 @@ base:
     {% endif %}
     - secrets
     - healthcheck.standalone
-    - idstools.soc_idstools
-    - idstools.adv_idstools
     - kratos.soc_kratos
     - kratos.adv_kratos
     - hydra.soc_hydra
@@ -172,7 +171,6 @@ base:
     - manager.adv_manager
     - soc.soc_soc
     - soc.adv_soc
-    - soc.license
     - kibana.soc_kibana
     - kibana.adv_kibana
     - strelka.soc_strelka
@@ -183,8 +181,6 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
-    - pcap.soc_pcap
-    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - minions.{{ grains.id }}
@@ -207,8 +203,6 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
-    - pcap.soc_pcap
-    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - strelka.soc_strelka
@@ -238,7 +232,6 @@ base:
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
-    - soc.license
     - kafka.nodes
     - kafka.soc_kafka
     - kafka.adv_kafka
@@ -256,10 +249,12 @@ base:
     - minions.adv_{{ grains.id }}
     - kafka.nodes
     - kafka.soc_kafka
-    - kafka.adv_kafka
-    - soc.license
+    - stig.soc_stig
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
 
   '*_import':
+    - node_data.ips
     - secrets
     - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
@@ -280,7 +275,6 @@ base:
     - manager.adv_manager
     - soc.soc_soc
     - soc.adv_soc
-    - soc.license
     - kibana.soc_kibana
     - kibana.adv_kibana
     - backup.soc_backup
@@ -295,8 +289,6 @@ base:
     - zeek.adv_zeek
     - bpf.soc_bpf
     - bpf.adv_bpf
-    - pcap.soc_pcap
-    - pcap.adv_pcap
     - suricata.soc_suricata
     - suricata.adv_suricata
     - strelka.soc_strelka
@@ -305,6 +297,7 @@ base:
     - minions.adv_{{ grains.id }}
 
   '*_fleet':
+    - node_data.ips
     - backup.soc_backup
     - backup.adv_backup
     - logstash.nodes
@@ -314,9 +307,15 @@ base:
     - elasticfleet.adv_elasticfleet
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - stig.soc_stig
+
+  '*_hypervisor':
+    - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
+    - stig.soc_stig
 
   '*_desktop':
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
-    - soc.license
+

salt/_modules/hypervisor.py

@@ -0,0 +1,91 @@
+#!/opt/saltstack/salt/bin/python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+"""
+Salt execution module for hypervisor operations.
+
+This module provides functions for managing hypervisor configurations,
+including VM file management.
+"""
+
+import json
+import logging
+import os
+
+log = logging.getLogger(__name__)
+
+__virtualname__ = 'hypervisor'
+
+
+def __virtual__():
+    """
+    Only load this module if we're on a system that can manage hypervisors.
+    """
+    return __virtualname__
+
+
+def remove_vm_from_vms_file(vms_file_path, vm_hostname, vm_role):
+    """
+    Remove a VM entry from the hypervisorVMs file.
+    
+    Args:
+        vms_file_path (str): Path to the hypervisorVMs file
+        vm_hostname (str): Hostname of the VM to remove (without role suffix)
+        vm_role (str): Role of the VM
+        
+    Returns:
+        dict: Result dictionary with success status and message
+        
+    CLI Example:
+        salt '*' hypervisor.remove_vm_from_vms_file /opt/so/saltstack/local/salt/hypervisor/hosts/hypervisor1VMs node1 nsm
+    """
+    try:
+        # Check if file exists
+        if not os.path.exists(vms_file_path):
+            msg = f"VMs file not found: {vms_file_path}"
+            log.error(msg)
+            return {'result': False, 'comment': msg}
+        
+        # Read current VMs
+        with open(vms_file_path, 'r') as f:
+            content = f.read().strip()
+            vms = json.loads(content) if content else []
+        
+        # Find and remove the VM entry
+        original_count = len(vms)
+        vms = [vm for vm in vms if not (vm.get('hostname') == vm_hostname and vm.get('role') == vm_role)]
+        
+        if len(vms) < original_count:
+            # VM was found and removed, write back to file
+            with open(vms_file_path, 'w') as f:
+                json.dump(vms, f, indent=2)
+            
+            # Set socore:socore ownership (939:939)
+            os.chown(vms_file_path, 939, 939)
+            
+            msg = f"Removed VM {vm_hostname}_{vm_role} from {vms_file_path}"
+            log.info(msg)
+            return {'result': True, 'comment': msg}
+        else:
+            msg = f"VM {vm_hostname}_{vm_role} not found in {vms_file_path}"
+            log.warning(msg)
+            return {'result': False, 'comment': msg}
+            
+    except json.JSONDecodeError as e:
+        msg = f"Failed to parse JSON in {vms_file_path}: {str(e)}"
+        log.error(msg)
+        return {'result': False, 'comment': msg}
+    except Exception as e:
+        msg = f"Failed to remove VM {vm_hostname}_{vm_role} from {vms_file_path}: {str(e)}"
+        log.error(msg)
+        return {'result': False, 'comment': msg}

salt/_modules/qcow2.py

@@ -0,0 +1,335 @@
+#!py
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+"""
+Salt module for managing QCOW2 image configurations and VM hardware settings. This module provides functions
+for modifying network configurations within QCOW2 images, adjusting virtual machine hardware settings, and
+creating virtual storage volumes. It serves as a Salt interface to the so-qcow2-modify-network,
+so-kvm-modify-hardware, and so-kvm-create-volume scripts.
+
+The module offers three main capabilities:
+1. Network Configuration: Modify network settings (DHCP/static IP) within QCOW2 images
+2. Hardware Configuration: Adjust VM hardware settings (CPU, memory, PCI passthrough)
+3. Volume Management: Create and attach virtual storage volumes for NSM data
+
+This module is intended to work with Security Onion's virtualization infrastructure and is typically
+used in conjunction with salt-cloud for VM provisioning and management.
+"""
+
+import logging
+import subprocess
+import shlex
+
+log = logging.getLogger(__name__)
+
+__virtualname__ = 'qcow2'
+
+def __virtual__():
+    return __virtualname__
+
+def modify_network_config(image, interface, mode, vm_name, ip4=None, gw4=None, dns4=None, search4=None):
+    '''
+    Usage:
+        salt '*' qcow2.modify_network_config image=<path> interface=<iface> mode=<mode> vm_name=<name> [ip4=<addr>] [gw4=<addr>] [dns4=<servers>] [search4=<domain>]
+
+    Options:
+        image
+            Path to the QCOW2 image file that will be modified
+        interface
+            Network interface name to configure (e.g., 'enp1s0')
+        mode
+            Network configuration mode, either 'dhcp4' or 'static4'
+        vm_name
+            Full name of the VM (hostname_role)
+        ip4
+            IPv4 address with CIDR notation (e.g., '192.168.1.10/24')
+            Required when mode='static4'
+        gw4
+            IPv4 gateway address (e.g., '192.168.1.1')
+            Required when mode='static4'
+        dns4
+            Comma-separated list of IPv4 DNS servers (e.g., '8.8.8.8,8.8.4.4')
+            Optional for both DHCP and static configurations
+        search4
+            DNS search domain for IPv4 (e.g., 'example.local')
+            Optional for both DHCP and static configurations
+
+    Examples:
+        1. **Configure DHCP:**
+            ```bash
+            salt '*' qcow2.modify_network_config image='/nsm/libvirt/images/sool9/sool9.qcow2' interface='enp1s0' mode='dhcp4'
+            ```
+            This configures enp1s0 to use DHCP for IP assignment
+
+        2. **Configure Static IP:**
+            ```bash
+            salt '*' qcow2.modify_network_config image='/nsm/libvirt/images/sool9/sool9.qcow2' interface='enp1s0' mode='static4' ip4='192.168.1.10/24' gw4='192.168.1.1' dns4='192.168.1.1,8.8.8.8' search4='example.local'
+            ```
+            This sets a static IP configuration with DNS servers and search domain
+
+    Notes:
+        - The QCOW2 image must be accessible and writable by the salt minion
+        - The image should not be in use by a running VM when modified
+        - Network changes take effect on next VM boot
+        - Requires so-qcow2-modify-network script to be installed
+
+    Description:
+        This function modifies network configuration within a QCOW2 image file by executing
+        the so-qcow2-modify-network script. It supports both DHCP and static IPv4 configuration.
+        The script mounts the image, modifies the network configuration files, and unmounts
+        safely. All operations are logged for troubleshooting purposes.
+
+    Exit Codes:
+        0: Success
+        1: Invalid parameters or configuration
+        2: Image access or mounting error
+        3: Network configuration error
+        4: System command error
+        255: Unexpected error
+
+    Logging:
+        - All operations are logged to the salt minion log
+        - Log entries are prefixed with 'qcow2 module:'
+        - Error conditions include detailed error messages and stack traces
+        - Success/failure status is logged for verification
+    '''
+
+    cmd = ['/usr/sbin/so-qcow2-modify-network', '-I', image, '-i', interface, '-n', vm_name]
+
+    if mode.lower() == 'dhcp4':
+        cmd.append('--dhcp4')
+    elif mode.lower() == 'static4':
+        cmd.append('--static4')
+        if not ip4 or not gw4:
+            raise ValueError('Both ip4 and gw4 are required for static configuration.')
+        cmd.extend(['--ip4', ip4, '--gw4', gw4])
+        if dns4:
+            cmd.extend(['--dns4', dns4])
+        if search4:
+            cmd.extend(['--search4', search4])
+    else:
+        raise ValueError("Invalid mode '{}'. Expected 'dhcp4' or 'static4'.".format(mode))
+
+    log.info('qcow2 module: Executing command: {}'.format(' '.join(shlex.quote(arg) for arg in cmd)))
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+        ret = {
+            'retcode': result.returncode,
+            'stdout': result.stdout,
+            'stderr': result.stderr
+        }
+        if result.returncode != 0:
+            log.error('qcow2 module: Script execution failed with return code {}: {}'.format(result.returncode, result.stderr))
+        else:
+            log.info('qcow2 module: Script executed successfully.')
+        return ret
+    except Exception as e:
+        log.error('qcow2 module: An error occurred while executing the script: {}'.format(e))
+        raise
+
+def modify_hardware_config(vm_name, cpu=None, memory=None, pci=None, start=False):
+    '''
+    Usage:
+        salt '*' qcow2.modify_hardware_config vm_name=<name> [cpu=<count>] [memory=<size>] [pci=<id>] [pci=<id>] [start=<bool>]
+
+    Options:
+        vm_name
+            Name of the virtual machine to modify
+        cpu
+            Number of virtual CPUs to assign (positive integer)
+            Optional - VM's current CPU count retained if not specified
+        memory
+            Amount of memory to assign in MiB (positive integer)
+            Optional - VM's current memory size retained if not specified
+        pci
+            PCI hardware ID(s) to passthrough to the VM (e.g., '0000:c7:00.0')
+            Can be specified multiple times for multiple devices
+            Optional - no PCI passthrough if not specified
+        start
+            Boolean flag to start the VM after modification
+            Optional - defaults to False
+
+    Examples:
+        1. **Modify CPU and Memory:**
+            ```bash
+            salt '*' qcow2.modify_hardware_config vm_name='sensor1' cpu=4 memory=8192
+            ```
+            This assigns 4 CPUs and 8GB memory to the VM
+
+        2. **Enable PCI Passthrough:**
+            ```bash
+            salt '*' qcow2.modify_hardware_config vm_name='sensor1' pci='0000:c7:00.0' pci='0000:c4:00.0' start=True
+            ```
+            This configures PCI passthrough and starts the VM
+
+        3. **Complete Hardware Configuration:**
+            ```bash
+            salt '*' qcow2.modify_hardware_config vm_name='sensor1' cpu=8 memory=16384 pci='0000:c7:00.0' start=True
+            ```
+            This sets CPU, memory, PCI passthrough, and starts the VM
+
+    Notes:
+        - VM must be stopped before modification unless only the start flag is set
+        - Memory is specified in MiB (1024 = 1GB)
+        - PCI devices must be available and not in use by the host
+        - CPU count should align with host capabilities
+        - Requires so-kvm-modify-hardware script to be installed
+
+    Description:
+        This function modifies the hardware configuration of a KVM virtual machine using
+        the so-kvm-modify-hardware script. It can adjust CPU count, memory allocation,
+        and PCI device passthrough. Changes are applied to the VM's libvirt configuration.
+        The VM can optionally be started after modifications are complete.
+
+    Exit Codes:
+        0: Success
+        1: Invalid parameters
+        2: VM state error (running when should be stopped)
+        3: Hardware configuration error
+        4: System command error
+        255: Unexpected error
+
+    Logging:
+        - All operations are logged to the salt minion log
+        - Log entries are prefixed with 'qcow2 module:'
+        - Hardware configuration changes are logged
+        - Errors include detailed messages and stack traces
+        - Final status of modification is logged
+    '''
+
+    cmd = ['/usr/sbin/so-kvm-modify-hardware', '-v', vm_name]
+
+    if cpu is not None:
+        if isinstance(cpu, int) and cpu > 0:
+            cmd.extend(['-c', str(cpu)])
+        else:
+            raise ValueError('cpu must be a positive integer.')
+    if memory is not None:
+        if isinstance(memory, int) and memory > 0:
+            cmd.extend(['-m', str(memory)])
+        else:
+            raise ValueError('memory must be a positive integer.')
+    if pci:
+        # Handle PCI IDs (can be a single device or comma-separated list)
+        if isinstance(pci, str):
+            devices = [dev.strip() for dev in pci.split(',') if dev.strip()]
+        elif isinstance(pci, list):
+            devices = pci
+        else:
+            devices = [pci]
+
+        # Add each device with its own -p flag
+        for device in devices:
+            cmd.extend(['-p', str(device)])
+    if start:
+        cmd.append('-s')
+
+    log.info('qcow2 module: Executing command: {}'.format(' '.join(shlex.quote(arg) for arg in cmd)))
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+        ret = {
+            'retcode': result.returncode,
+            'stdout': result.stdout,
+            'stderr': result.stderr
+        }
+        if result.returncode != 0:
+            log.error('qcow2 module: Script execution failed with return code {}: {}'.format(result.returncode, result.stderr))
+        else:
+            log.info('qcow2 module: Script executed successfully.')
+        return ret
+    except Exception as e:
+        log.error('qcow2 module: An error occurred while executing the script: {}'.format(e))
+        raise
+
+def create_volume_config(vm_name, size_gb, start=False):
+    '''
+    Usage:
+        salt '*' qcow2.create_volume_config vm_name=<name> size_gb=<size> [start=<bool>]
+
+    Options:
+        vm_name
+            Name of the virtual machine to attach the volume to
+        size_gb
+            Volume size in GB (positive integer)
+            This determines the capacity of the virtual storage volume
+        start
+            Boolean flag to start the VM after volume creation
+            Optional - defaults to False
+
+    Examples:
+        1. **Create 500GB Volume:**
+            ```bash
+            salt '*' qcow2.create_volume_config vm_name='sensor1_sensor' size_gb=500
+            ```
+            This creates a 500GB virtual volume for NSM storage
+
+        2. **Create 1TB Volume and Start VM:**
+            ```bash
+            salt '*' qcow2.create_volume_config vm_name='sensor1_sensor' size_gb=1000 start=True
+            ```
+            This creates a 1TB volume and starts the VM after attachment
+
+    Notes:
+        - VM must be stopped before volume creation
+        - Volume is created as a qcow2 image and attached to the VM
+        - This is an alternative to disk passthrough via modify_hardware_config
+        - Volume is automatically attached to the VM's libvirt configuration
+        - Requires so-kvm-create-volume script to be installed
+        - Volume files are stored in the hypervisor's VM storage directory
+
+    Description:
+        This function creates and attaches a virtual storage volume to a KVM virtual machine
+        using the so-kvm-create-volume script. It creates a qcow2 disk image of the specified
+        size and attaches it to the VM for NSM (Network Security Monitoring) storage purposes.
+        This provides an alternative to physical disk passthrough, allowing flexible storage
+        allocation without requiring dedicated hardware. The VM can optionally be started
+        after the volume is successfully created and attached.
+
+    Exit Codes:
+        0: Success
+        1: Invalid parameters
+        2: VM state error (running when should be stopped)
+        3: Volume creation error
+        4: System command error
+        255: Unexpected error
+
+    Logging:
+        - All operations are logged to the salt minion log
+        - Log entries are prefixed with 'qcow2 module:'
+        - Volume creation and attachment operations are logged
+        - Errors include detailed messages and stack traces
+        - Final status of volume creation is logged
+    '''
+
+    # Validate size_gb parameter
+    if not isinstance(size_gb, int) or size_gb <= 0:
+        raise ValueError('size_gb must be a positive integer.')
+
+    cmd = ['/usr/sbin/so-kvm-create-volume', '-v', vm_name, '-s', str(size_gb)]
+
+    if start:
+        cmd.append('-S')
+
+    log.info('qcow2 module: Executing command: {}'.format(' '.join(shlex.quote(arg) for arg in cmd)))
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+        ret = {
+            'retcode': result.returncode,
+            'stdout': result.stdout,
+            'stderr': result.stderr
+        }
+        if result.returncode != 0:
+            log.error('qcow2 module: Script execution failed with return code {}: {}'.format(result.returncode, result.stderr))
+        else:
+            log.info('qcow2 module: Script executed successfully.')
+        return ret
+    except Exception as e:
+        log.error('qcow2 module: An error occurred while executing the script: {}'.format(e))
+        raise

salt/allowed_states.map.jinja

@@ -1,264 +1,160 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
 
 {% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %}
 {% import_yaml 'salt/minion.defaults.yaml' as saltversion %}
 {% set saltversion = saltversion.salt.minion.version %}
 
-{# this is the list we are returning from this map file, it gets built below #}
-{% set allowed_states= [] %}
+{# Define common state groups to reduce redundancy #}
+{% set base_states = [
+    'common',
+    'patch.os.schedule',
+    'motd',
+    'salt.minion-check',
+    'sensoroni',
+    'salt.lasthighstate',
+    'salt.minion',
+    'telegraf',
+    'firewall',
+    'schedule',
+    'docker_clean'
+] %}
+
+{% set manager_states = [
+    'salt.master',
+    'ca.server',
+    'registry',
+    'manager',
+    'nginx',
+    'influxdb',
+    'soc',
+    'kratos',
+    'hydra',
+    'elasticfleet',
+    'elastic-fleet-package-registry',
+    'utility'
+] %}
+
+{% set sensor_states = [
+    'suricata',
+    'healthcheck',
+    'tcpreplay',
+    'zeek',
+    'strelka'
+] %}
+
+{% set kafka_states = [
+    'kafka'
+] %}
+
+{% set stig_states = [
+    'stig'
+] %}
+
+{% set elastic_stack_states = [
+    'elasticsearch',
+    'elasticsearch.auth',
+    'kibana',
+    'kibana.secrets',
+    'elastalert',
+    'logstash',
+    'redis'
+] %}
+
+{# Initialize the allowed_states list #}
+{% set allowed_states = [] %}
 
 {% if grains.saltversion | string == saltversion | string %}
+    {# Map role-specific states #}
+    {% set role_states = {
+        'so-eval': (
+            manager_states +
+            sensor_states +
+            elastic_stack_states | reject('equalto', 'logstash') | list +
+            ['logstash.ssl']
+        ),
+        'so-heavynode': (
+            sensor_states +
+            ['elasticagent', 'elasticsearch', 'logstash', 'redis', 'nginx']
+        ),
+        'so-idh': (
+            ['idh']
+        ),
+        'so-import': (
+            manager_states +
+            sensor_states | reject('equalto', 'strelka') | reject('equalto', 'healthcheck') | list +
+            ['elasticsearch', 'elasticsearch.auth', 'kibana', 'kibana.secrets', 'logstash.ssl', 'strelka.manager']
+        ),
+        'so-manager': (
+            manager_states +
+            ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users', 'strelka.manager'] +
+            stig_states +
+            kafka_states +
+            elastic_stack_states
+        ),
+        'so-managerhype': (
+            manager_states +
+            ['salt.cloud', 'strelka.manager', 'hypervisor', 'libvirt'] +
+            stig_states +
+            kafka_states +
+            elastic_stack_states
+        ),
+        'so-managersearch': (
+            manager_states +
+            ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users', 'strelka.manager'] +
+            stig_states +
+            kafka_states +
+            elastic_stack_states
+        ),
+        'so-searchnode': (
+            ['kafka.ca', 'kafka.ssl', 'elasticsearch', 'logstash', 'nginx'] +
+            stig_states
+        ),
+        'so-standalone': (
+            manager_states +
+            ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users'] +
+            sensor_states +
+            stig_states +
+            kafka_states +
+            elastic_stack_states
+        ),
+        'so-sensor': (
+            sensor_states +
+            ['nginx'] +
+            stig_states
+        ),
+        'so-fleet': (
+            stig_states +
+            ['logstash', 'nginx', 'healthcheck', 'elasticfleet']
+        ),
+        'so-receiver': (
+            kafka_states +
+            stig_states +
+            ['logstash', 'redis']
+        ),
+        'so-hypervisor': (
+            stig_states +
+            ['hypervisor', 'libvirt']
+        ),
+        'so-desktop': (
+            stig_states
+        )
+    } %}
 
-  {% set allowed_states= salt['grains.filter_by']({
-      'so-eval': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'manager',
-          'nginx',
-          'telegraf',
-          'influxdb',
-          'soc',
-          'kratos',
-          'hydra',
-          'elasticfleet',
-          'elastic-fleet-package-registry',
-          'firewall',
-          'idstools',
-          'suricata.manager',
-          'healthcheck',
-          'pcap',
-          'suricata',
-          'utility',
-          'schedule',
-          'tcpreplay',
-          'docker_clean'
-          ],
-      'so-heavynode': [
-          'ssl',
-          'nginx',
-          'telegraf',
-          'firewall',
-          'pcap',
-          'suricata',
-          'healthcheck',
-          'elasticagent',
-          'schedule',
-          'tcpreplay',
-          'docker_clean'
-          ],
-     'so-idh': [
-          'ssl',
-          'telegraf',
-          'firewall',
-          'idh',
-          'schedule',
-          'docker_clean'
-          ],
-      'so-import': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'manager',
-          'nginx',
-          'strelka.manager',
-          'soc',
-          'kratos',
-          'hydra',
-          'influxdb',
-          'telegraf',
-          'firewall',
-          'idstools',
-          'suricata.manager',
-          'pcap',
-          'utility',
-          'suricata',
-          'zeek',
-          'schedule',
-          'tcpreplay',
-          'docker_clean',
-          'elasticfleet',
-          'elastic-fleet-package-registry'
-          ],
-      'so-manager': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'manager',
-          'nginx',
-          'telegraf',
-          'influxdb',
-          'strelka.manager',
-          'soc',
-          'kratos',
-          'hydra',
-          'elasticfleet',
-          'elastic-fleet-package-registry',
-          'firewall',
-          'idstools',
-          'suricata.manager',
-          'utility',
-          'schedule',
-          'docker_clean',
-          'stig',
-          'kafka'
-          ],
-      'so-managersearch': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'nginx',
-          'telegraf',
-          'influxdb',
-          'strelka.manager',
-          'soc',
-          'kratos',
-          'hydra',
-          'elastic-fleet-package-registry',
-          'elasticfleet',
-          'firewall',
-          'manager',
-          'idstools',
-          'suricata.manager',
-          'utility',
-          'schedule',
-          'docker_clean',
-          'stig',
-          'kafka'
-          ],
-      'so-searchnode': [
-          'ssl',
-          'nginx',
-          'telegraf',
-          'firewall',
-          'schedule',
-          'docker_clean',
-          'stig',
-          'kafka.ca',
-          'kafka.ssl'
-          ],
-      'so-standalone': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'manager',
-          'nginx',
-          'telegraf',
-          'influxdb',
-          'soc',
-          'kratos',
-          'hydra',
-          'elastic-fleet-package-registry',
-          'elasticfleet',
-          'firewall',
-          'idstools',
-          'suricata.manager',
-          'pcap',
-          'suricata',
-          'healthcheck',
-          'utility',
-          'schedule',
-          'tcpreplay',
-          'docker_clean',
-          'stig',
-          'kafka'
-          ],
-      'so-sensor': [
-          'ssl',
-          'telegraf',
-          'firewall',
-          'nginx',
-          'pcap',
-          'suricata',
-          'healthcheck',
-          'schedule',
-          'tcpreplay',
-          'docker_clean',
-          'stig'
-          ],
-      'so-fleet': [
-          'ssl',
-          'telegraf',
-          'firewall',
-          'logstash',
-          'nginx',
-          'healthcheck',
-          'schedule',
-          'elasticfleet',
-          'docker_clean'
-          ],
-      'so-receiver': [
-          'ssl',
-          'telegraf',
-          'firewall',
-          'schedule',
-          'docker_clean',
-          'kafka',
-          'stig'
-          ],
-      'so-desktop': [
-          'ssl',
-          'docker_clean',
-          'telegraf',
-          'stig'
-          ],
-  }, grain='role') %}
-
-  {%- if grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
-    {% do allowed_states.append('zeek') %}
-  {%- endif %}
-
-  {% if grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
-    {% do allowed_states.append('strelka') %}
-  {% endif %}
-
-  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import'] %}
-    {% do allowed_states.append('elasticsearch') %}
-  {% endif %}
-
-  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
-    {% do allowed_states.append('elasticsearch.auth') %}
-  {% endif %}
-
-  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
-    {% do allowed_states.append('kibana') %}
-    {% do allowed_states.append('kibana.secrets') %}
-  {% endif %}
-
-  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
-    {% do allowed_states.append('elastalert') %}
-  {% endif %}
-
-  {% if grains.role in ['so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
-    {% do allowed_states.append('logstash') %}
-  {% endif %}
-
-  {% if grains.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-receiver', 'so-eval'] %}
-    {% do allowed_states.append('redis') %}
-  {% endif %}
-  
-  {# all nodes on the right salt version can run the following states #}
-  {% do allowed_states.append('common') %}
-  {% do allowed_states.append('patch.os.schedule') %}
-  {% do allowed_states.append('motd') %}
-  {% do allowed_states.append('salt.minion-check') %}
-  {% do allowed_states.append('sensoroni') %}
-  {% do allowed_states.append('salt.lasthighstate') %}
+    {# Get states for the current role #}
+    {% if grains.role in role_states %}
+        {% set allowed_states = role_states[grains.role] %}
+    {% endif %}
 
+    {# Add base states that apply to all roles #}
+    {% for state in base_states %}
+        {% do allowed_states.append(state) %}
+    {% endfor %}
 {% endif %}
 
-
+{# Add airgap state if needed #}
 {% if ISAIRGAP %}
-  {% do allowed_states.append('airgap') %}
+    {% do allowed_states.append('airgap') %}
 {% endif %}
-
-{# all nodes can always run salt.minion state #}
-{% do allowed_states.append('salt.minion') %}

salt/backup/tools/sbin/so-config-backup.jinja

@@ -11,6 +11,10 @@ TODAY=$(date '+%Y_%m_%d')
 BACKUPDIR={{ DESTINATION }}
 BACKUPFILE="$BACKUPDIR/so-config-backup-$TODAY.tar"
 MAXBACKUPS=7
+EXCLUSIONS=(
+  "--exclude=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers"
+)
+
 
 # Create backup dir if it does not exist
 mkdir -p /nsm/backup
@@ -23,7 +27,7 @@ if [ ! -f $BACKUPFILE ]; then
 
   # Loop through all paths defined in global.sls, and append them to backup file
   {%- for LOCATION in BACKUPLOCATIONS %}
-  tar -rf $BACKUPFILE {{ LOCATION }}
+  tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }}
   {%- endfor %}
 
 fi

salt/bpf/macros.jinja

@@ -1,10 +1,12 @@
 {% macro remove_comments(bpfmerged, app) %}
 
 {# remove comments from the bpf #}
+{% set app_list = [] %}
 {% for bpf in bpfmerged[app] %}
-{%   if bpf.strip().startswith('#') %}
-{%     do bpfmerged[app].pop(loop.index0) %}
+{%   if not bpf.strip().startswith('#') %}
+{%     do app_list.append(bpf) %}
 {%   endif %}
 {% endfor %}
+{% do bpfmerged.update({app: app_list}) %}
 
 {% endmacro %}

salt/bpf/pcap.map.jinja

@@ -1,10 +1,15 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
-{% if GLOBALS.pcap_engine == "TRANSITION" %}
-{%   set PCAPBPF = ["ip and host 255.255.255.1 and port 1"] %}
-{% else %}
+{% set PCAP_BPF_STATUS = 0  %}
+
 {%   import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {%   set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
 {%   import 'bpf/macros.jinja' as MACROS %}
 {{   MACROS.remove_comments(BPFMERGED, 'pcap') }}
 {%   set PCAPBPF = BPFMERGED.pcap %}
+
+{% if PCAPBPF %}
+   {% set PCAP_BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + PCAPBPF|join(" "),cwd='/root') %}
+   {% if PCAP_BPF_CALC['retcode'] == 0 %}
+      {% set PCAP_BPF_STATUS = 1 %}
+   {% endif %}
 {% endif %}

salt/bpf/soc_bpf.yaml

@@ -1,11 +1,11 @@
 bpf:
   pcap:
-    description: List of BPF filters to apply to Stenographer.
+    description: List of BPF filters to apply to the PCAP engine.
     multiline: True
     forcedType: "[]string"
     helpLink: bpf.html
   suricata:
-    description: List of BPF filters to apply to Suricata.
+    description: List of BPF filters to apply to Suricata. This will apply to alerts and, if enabled, to metadata and PCAP logs generated by Suricata.
     multiline: True
     forcedType: "[]string"
     helpLink: bpf.html

salt/bpf/suricata.map.jinja

@@ -1,7 +1,16 @@
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
+{% set SURICATA_BPF_STATUS = 0 %}
 {% import 'bpf/macros.jinja' as MACROS %}
 
 {{ MACROS.remove_comments(BPFMERGED, 'suricata') }}
 
 {% set SURICATABPF = BPFMERGED.suricata %}
+
+{% if SURICATABPF %}
+   {% set SURICATA_BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + SURICATABPF|join(" "),cwd='/root') %}
+   {% if SURICATA_BPF_CALC['retcode'] == 0 %}
+      {% set SURICATA_BPF_STATUS = 1 %}
+   {% endif %}
+{% endif %}

salt/bpf/zeek.map.jinja

@@ -1,7 +1,16 @@
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
+{% set ZEEK_BPF_STATUS = 0 %}
 {% import 'bpf/macros.jinja' as MACROS %}
 
 {{ MACROS.remove_comments(BPFMERGED, 'zeek') }}
 
 {% set ZEEKBPF = BPFMERGED.zeek %}
+
+{% if ZEEKBPF %}
+   {% set ZEEK_BPF_CALC = salt['cmd.script']('salt://common/tools/sbin/so-bpf-compile', GLOBALS.sensor.interface + ' ' + ZEEKBPF|join(" "),cwd='/root') %}
+   {% if ZEEK_BPF_CALC['retcode'] == 0 %}
+      {% set ZEEK_BPF_STATUS = 1 %}
+   {% endif %}
+{% endif %}

salt/ca/dirs.sls

@@ -1,4 +0,0 @@
-pki_issued_certs:
-  file.directory:
-    - name: /etc/pki/issued_certs
-    - makedirs: True

salt/ca/init.sls

@@ -3,70 +3,10 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
-
 include:
-  - ca.dirs
-
-/etc/salt/minion.d/signing_policies.conf:
-  file.managed:
-    - source: salt://ca/files/signing_policies.conf
-
-pki_private_key:
-  x509.private_key_managed:
-    - name: /etc/pki/ca.key
-    - keysize: 4096
-    - passphrase:
-    - backup: True
-    {% if salt['file.file_exists']('/etc/pki/ca.key') -%}
-    - prereq:
-      - x509: /etc/pki/ca.crt
-    {%- endif %}
-
-pki_public_ca_crt:
-  x509.certificate_managed:
-    - name: /etc/pki/ca.crt
-    - signing_private_key: /etc/pki/ca.key
-    - CN: {{ GLOBALS.manager }}
-    - C: US
-    - ST: Utah
-    - L: Salt Lake City
-    - basicConstraints: "critical CA:true"
-    - keyUsage: "critical cRLSign, keyCertSign"
-    - extendedkeyUsage: "serverAuth, clientAuth"
-    - subjectKeyIdentifier: hash
-    - authorityKeyIdentifier: keyid:always, issuer
-    - days_valid: 3650
-    - days_remaining: 0
-    - backup: True
-    - replace: False
-    - require:
-      - sls: ca.dirs
-    - timeout: 30
-    - retry:
-        attempts: 5
-        interval: 30
-
-mine_update_ca_crt:
-  module.run:
-    - mine.update: []
-    - onchanges:
-      - x509: pki_public_ca_crt
-
-cakeyperms:
-  file.managed:
-    - replace: False
-    - name: /etc/pki/ca.key
-    - mode: 640
-    - group: 939
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
+{% if GLOBALS.is_manager %}
+  - ca.server
 {% endif %}
+  - ca.trustca

salt/ca/map.jinja

@@ -0,0 +1,3 @@
+{% set CA = {
+    'server': pillar.ca.server
+}%}

salt/ca/remove.sls

@@ -1,7 +1,35 @@
-pki_private_key:
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% set setup_running = salt['cmd.retcode']('pgrep -x so-setup') == 0 %}
+
+{% if setup_running%}
+
+include:
+  - ssl.remove
+
+remove_pki_private_key:
   file.absent:
     - name: /etc/pki/ca.key
 
-pki_public_ca_crt:
+remove_pki_public_ca_crt:
   file.absent:
     - name: /etc/pki/ca.crt
+
+remove_trusttheca:
+  file.absent:
+    - name: /etc/pki/tls/certs/intca.crt
+
+remove_pki_public_ca_crt_symlink:
+  file.absent:
+    - name: /opt/so/saltstack/local/salt/ca/files/ca.crt
+
+{% else %}
+
+so-setup_not_running:
+  test.show_notification:
+    - text: "This state is reserved for usage during so-setup."
+
+{% endif %}

salt/ca/server.sls

@@ -0,0 +1,63 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+
+pki_private_key:
+  x509.private_key_managed:
+    - name: /etc/pki/ca.key
+    - keysize: 4096
+    - passphrase:
+    - backup: True
+    {% if salt['file.file_exists']('/etc/pki/ca.key') -%}
+    - prereq:
+      - x509: /etc/pki/ca.crt
+    {%- endif %}
+
+pki_public_ca_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/ca.crt
+    - signing_private_key: /etc/pki/ca.key
+    - CN: {{ GLOBALS.manager }}
+    - C: US
+    - ST: Utah
+    - L: Salt Lake City
+    - basicConstraints: "critical CA:true"
+    - keyUsage: "critical cRLSign, keyCertSign"
+    - extendedkeyUsage: "serverAuth, clientAuth"
+    - subjectKeyIdentifier: hash
+    - authorityKeyIdentifier: keyid:always, issuer
+    - days_valid: 3650
+    - days_remaining: 7
+    - backup: True
+    - replace: False
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+
+pki_public_ca_crt_symlink:
+  file.symlink:
+    - name: /opt/so/saltstack/local/salt/ca/files/ca.crt
+    - target: /etc/pki/ca.crt
+    - require:
+      - x509: pki_public_ca_crt
+
+cakeyperms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/ca.key
+    - mode: 640
+    - group: 939
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/ca/signing_policy.sls

@@ -1,12 +1,15 @@
-#!/bin/bash
-
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+# when the salt-minion signs the cert, a copy is stored here
+issued_certs_copypath:
+  file.directory:
+    - name: /etc/pki/issued_certs
+    - makedirs: True
 
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-start idstools $1
+signing_policy:
+  file.managed:
+    - name: /etc/salt/minion.d/signing_policies.conf
+    - source: salt://ca/files/signing_policies.conf

salt/ca/trustca.sls

@@ -0,0 +1,26 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+include:
+  - docker
+
+# Trust the CA
+trusttheca:
+  file.managed:
+    - name: /etc/pki/tls/certs/intca.crt
+    - source: salt://ca/files/ca.crt
+    - watch_in:
+      - service: docker_running
+    - show_changes: False
+    - makedirs: True
+
+{% if GLOBALS.os_family == 'Debian' %}
+symlinkca:
+  file.symlink:
+    - target: /etc/pki/tls/certs/intca.crt
+    - name: /etc/ssl/certs/intca.crt
+{% endif %}

salt/common/grains.sls

@@ -0,0 +1,21 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% set nsm_exists = salt['file.directory_exists']('/nsm') %}
+{% if nsm_exists %}
+{%   set nsm_total = salt['cmd.shell']('df -BG /nsm | tail -1 | awk \'{print $2}\'') %}
+
+nsm_total:
+  grains.present:
+    - name: nsm_total
+    - value: {{ nsm_total }}
+
+{% else %}
+
+nsm_missing:
+  test.succeed_without_changes:
+    - name: /nsm does not exist, skipping grain assignment
+
+{% endif %}

salt/common/init.sls

@@ -4,6 +4,7 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
 include:
+  - common.grains
   - common.packages
 {% if GLOBALS.role in GLOBALS.manager_roles %}
   - manager.elasticsearch # needed for elastic_curl_config state
@@ -106,7 +107,7 @@ Etc/UTC:
   timezone.system
 
 # Sync curl configuration for Elasticsearch authentication
-{% if GLOBALS.role in ['so-eval', 'so-heavynode', 'so-import', 'so-manager', 'so-managersearch', 'so-searchnode', 'so-standalone'] %}
+{% if GLOBALS.is_manager or GLOBALS.role in ['so-heavynode', 'so-searchnode'] %}
 elastic_curl_config:
   file.managed:
     - name: /opt/so/conf/elasticsearch/curl.config
@@ -129,6 +130,10 @@ common_sbin:
     - group: 939
     - file_mode: 755
     - show_changes: False
+{% if GLOBALS.role == 'so-heavynode' %}
+    - exclude_pat:
+      - so-pcap-import
+{% endif %}
 
 common_sbin_jinja:
   file.recurse:
@@ -139,6 +144,20 @@ common_sbin_jinja:
     - file_mode: 755
     - template: jinja
     - show_changes: False
+{% if GLOBALS.role == 'so-heavynode' %}
+    - exclude_pat:
+      - so-import-pcap
+{% endif %}
+
+{% if GLOBALS.role == 'so-heavynode' %}
+remove_so-pcap-import_heavynode:
+  file.absent:
+    - name: /usr/sbin/so-pcap-import
+
+remove_so-import-pcap_heavynode:
+  file.absent:
+    - name: /usr/sbin/so-import-pcap
+{% endif %}
 
 {% if not GLOBALS.is_manager%}
 # prior to 2.4.50 these scripts were in common/tools/sbin on the manager because of soup and distributed to non managers
@@ -158,7 +177,7 @@ so-status_script:
     - source: salt://common/tools/sbin/so-status
     - mode: 755
 
-{% if GLOBALS.role in GLOBALS.sensor_roles %}
+{% if GLOBALS.is_sensor %}
 # Add sensor cleanup
 so-sensor-clean:
   cron.present:

salt/common/packages.sls

@@ -1,6 +1,6 @@
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
-{% if GLOBALS.os_family == 'Debian' %}
+# we cannot import GLOBALS from vars/globals.map.jinja in this state since it is called in setup.virt.init
+# since it is early in setup of a new VM, the pillars imported in GLOBALS are not yet defined
+{% if grains.os_family == 'Debian' %}
 commonpkgs:
   pkg.installed:
     - skip_suggestions: True
@@ -46,7 +46,7 @@ python-rich:
 {%     endif %}
 {% endif %}
 
-{% if GLOBALS.os_family == 'RedHat' %}
+{% if grains.os_family == 'RedHat' %}
 
 remove_mariadb:
   pkg.removed:

salt/common/soup_scripts.sls

@@ -3,8 +3,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{% if '2.4' in salt['cp.get_file_str']('/etc/soversion') %}
-
 {%   import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %}
 {%   if SOC_GLOBAL.global.airgap %}
 {%     set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %}
@@ -120,23 +118,3 @@ copy_bootstrap-salt_sbin:
     - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
     - force: True
     - preserve: True
-
-{# this is added in 2.4.120 to remove salt repo files pointing to saltproject.io to accomodate the move to broadcom and new bootstrap-salt script #}
-{%   if salt['pkg.version_cmp'](SOVERSION, '2.4.120') == -1 %}
-{%     set saltrepofile = '/etc/yum.repos.d/salt.repo' %}
-{%     if grains.os_family == 'Debian' %}
-{%       set saltrepofile = '/etc/apt/sources.list.d/salt.list' %}
-{%     endif %}
-remove_saltproject_io_repo_manager:
-  file.absent:
-    - name: {{ saltrepofile }}
-{%   endif %}
-
-{% else %}
-fix_23_soup_sbin:
-  cmd.run:
-    - name: curl -s -f -o /usr/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
-fix_23_soup_salt:
-  cmd.run:
-    - name: curl -s -f -o /opt/so/saltstack/defalt/salt/common/tools/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
-{% endif %}

salt/common/tools/sbin/so-bpf-compile

@@ -16,7 +16,7 @@
 
 if [ "$#" -lt 2 ]; then
   cat 1>&2 <<EOF
-$0 compiles a BPF expression to be passed to stenotype to apply a socket filter.
+$0 compiles a BPF expression to be passed to PCAP to apply a socket filter.
 Its first argument is the interface (link type is required) and all other arguments
 are passed to TCPDump.
 
@@ -29,9 +29,26 @@ fi
 
 interface="$1"
 shift
-tcpdump -i $interface -ddd $@ | tail -n+2 |
-while read line; do
+
+# Capture tcpdump output and exit code
+tcpdump_output=$(tcpdump -i "$interface" -ddd "$@" 2>&1)
+tcpdump_exit=$?
+
+if [ $tcpdump_exit -ne 0 ]; then
+  echo "$tcpdump_output" >&2
+  exit $tcpdump_exit
+fi
+
+# Process the output, skipping the first line
+echo "$tcpdump_output" | tail -n+2 | while read -r line; do
   cols=( $line )
-  printf "%04x%02x%02x%08x" ${cols[0]} ${cols[1]} ${cols[2]} ${cols[3]}
+  printf "%04x%02x%02x%08x" "${cols[0]}" "${cols[1]}" "${cols[2]}" "${cols[3]}"
 done
+
+# Check if the pipeline succeeded
+if [ "${PIPESTATUS[0]}" -ne 0 ]; then
+  exit 1
+fi
+
 echo ""
+exit 0

salt/common/tools/sbin/so-checkin

@@ -10,7 +10,7 @@
 cat << EOF
 
 so-checkin will run a full salt highstate to apply all salt states. If a highstate is already running, this request will be queued and so it may pause for a few minutes before you see any more output. For more information about so-checkin and salt, please see:
-https://docs.securityonion.net/en/2.4/salt.html
+https://securityonion.net/docs/salt
 
 EOF
 

salt/common/tools/sbin/so-common

@@ -10,7 +10,7 @@
 # and since this same logic is required during installation, it's included in this file.
 
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
-DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
+DOC_BASE_URL="https://securityonion.net/docs"
 
 if [ -z $NOROOT ]; then
 	# Check for prerequisites
@@ -99,6 +99,17 @@ add_interface_bond0() {
 	fi
 }
 
+airgap_playbooks() {
+	SRC_DIR=$1
+	# Copy playbooks if using airgap
+	mkdir -p /nsm/airgap-resources
+	# Purge old airgap playbooks to ensure SO only uses the latest released playbooks
+	rm -fr /nsm/airgap-resources/playbooks
+	tar xf $SRC_DIR/airgap-resources/playbooks.tgz -C /nsm/airgap-resources/
+	chown -R socore:socore /nsm/airgap-resources/playbooks
+	git config --global --add safe.directory /nsm/airgap-resources/playbooks
+}
+
 check_container() {
 	docker ps | grep "$1:" > /dev/null 2>&1
 	return $?
@@ -209,12 +220,22 @@ compare_es_versions() {
 }
 
 copy_new_files() {
+  # Define files to exclude from deletion (relative to their respective base directories)
+  local EXCLUDE_FILES=(
+    "salt/hypervisor/soc_hypervisor.yaml"
+  )
+  
+  # Build rsync exclude arguments
+  local EXCLUDE_ARGS=()
+  for file in "${EXCLUDE_FILES[@]}"; do
+    EXCLUDE_ARGS+=(--exclude="$file")
+  done
+  
   # Copy new files over to the salt dir
   cd $UPDATE_DIR
-  rsync -a salt $DEFAULT_SALT_DIR/ --delete
-  rsync -a pillar $DEFAULT_SALT_DIR/ --delete
+  rsync -a salt $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
+  rsync -a pillar $DEFAULT_SALT_DIR/ --delete "${EXCLUDE_ARGS[@]}"
   chown -R socore:socore $DEFAULT_SALT_DIR/
-  chmod 755 $DEFAULT_SALT_DIR/pillar/firewall/addfirewall.sh
   cd /tmp
 }
 
@@ -299,7 +320,8 @@ fail() {
 
 get_agent_count() {
 	if [ -f /opt/so/log/agents/agentstatus.log ]; then
-		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}')
+		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}' | sed 's/,//')
+		[[ -z "$AGENTCOUNT" ]] && AGENTCOUNT="0"
 	else
 		AGENTCOUNT=0
 	fi
@@ -311,8 +333,8 @@ get_elastic_agent_vars() {
 
 	if [ -f "$defaultsfile" ]; then
 		ELASTIC_AGENT_TARBALL_VERSION=$(egrep " +version: " $defaultsfile | awk -F: '{print $2}' | tr -d '[:space:]')
-		ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
-		ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+		ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/3/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+		ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/3/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
 		ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
 		ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
 		ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
@@ -373,7 +395,7 @@ is_manager_node() {
 }
 
 is_sensor_node() {
-	# Check to see if this is a sensor (forward) node
+	# Check to see if this is a sensor node
 	is_single_node_grid && return 0
 	grep "role: so-" /etc/salt/grains | grep -E "sensor|heavynode" &> /dev/null
 }
@@ -382,6 +404,25 @@ is_single_node_grid() {
 	grep "role: so-" /etc/salt/grains | grep -E "eval|standalone|import" &> /dev/null
 }
 
+initialize_elasticsearch_indices() {
+    local index_names=$1
+    local default_entry=${2:-'{"@timestamp":"0"}'}
+
+    for idx in $index_names; do
+        if ! so-elasticsearch-query "$idx" --fail --retry 3 --retry-delay 30 >/dev/null 2>&1; then
+            echo "Index does not already exist. Initializing $idx index."
+
+            if retry 3 10 "so-elasticsearch-query "$idx/_doc" -d '$default_entry' -XPOST --fail 2>/dev/null" '"successful":1'; then
+                echo "Successfully initialized $idx index."
+            else
+                echo "Failed to initialize $idx index after 3 attempts."
+            fi
+        else
+            echo "Index $idx already exists. No action needed."
+        fi
+    done
+}
+
 lookup_bond_interfaces() {
 	cat /proc/net/bonding/bond0 | grep "Slave Interface:" | sed -e "s/Slave Interface: //g"
 }
@@ -429,8 +470,7 @@ lookup_grain() {
 
 lookup_role() {
 	id=$(lookup_grain id)
-	pieces=($(echo $id | tr '_' ' '))
-	echo ${pieces[1]}
+	echo "${id##*_}"
 }
 
 is_feature_enabled() {
@@ -533,21 +573,39 @@ run_check_net_err() {
 }
 
 wait_for_salt_minion() {
-	local minion="$1"
-	local timeout="${2:-5}"
-	local logfile="${3:-'/dev/stdout'}"
-	retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$logfile" 2>&1 || fail
-	local attempt=0
-	# each attempts would take about 15 seconds
-	local maxAttempts=20
-	until check_salt_minion_status "$minion" "$timeout" "$logfile"; do
-		attempt=$((attempt+1))
-		if [[ $attempt -eq $maxAttempts ]]; then
-			return 1
-		fi
-		sleep 10
-	done
-	return 0
+    local minion="$1"
+    local max_wait="${2:-30}"
+    local interval="${3:-2}"
+    local logfile="${4:-'/dev/stdout'}"
+    local elapsed=0
+
+	echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Waiting for salt-minion '$minion' to be ready..."
+
+    while [ $elapsed -lt $max_wait ]; do
+        # Check if service is running
+		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Check if salt-minion service is running"
+        if ! systemctl is-active --quiet salt-minion; then
+            echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion service not running (elapsed: ${elapsed}s)"
+            sleep $interval
+            elapsed=$((elapsed + interval))
+            continue
+        fi
+		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion service is running"
+
+        # Check if minion responds to ping
+		echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Check if $minion responds to ping"
+        if salt "$minion" test.ping --timeout=3 --out=json 2>> "$logfile" | grep -q "true"; then
+            echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - salt-minion '$minion' is connected and ready!"
+            return 0
+        fi
+        
+        echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - Waiting... (${elapsed}s / ${max_wait}s)"
+        sleep $interval
+        elapsed=$((elapsed + interval))
+    done
+
+    echo "$(date '+%a %d %b %Y %H:%M:%S.%6N') - ERROR: salt-minion '$minion' not ready after $max_wait seconds"
+    return 1
 }
 
 salt_minion_count() {

salt/common/tools/sbin/so-common-status-check

@@ -45,7 +45,7 @@ def check_for_fps():
         result = subprocess.run([feat_full + '-mode-setup', '--is-enabled'], stdout=subprocess.PIPE)
         if result.returncode == 0:
             fps = 1
-    except FileNotFoundError:
+    except:
         fn = '/proc/sys/crypto/' + feat_full + '_enabled'
         try:
           with open(fn, 'r') as f:

salt/common/tools/sbin/so-docker-prune

@@ -4,22 +4,16 @@
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
-
-
-
-import sys, argparse, re, docker
+import sys, argparse, re, subprocess, json
 from packaging.version import Version, InvalidVersion
 from itertools import groupby, chain
 
-
 def get_image_name(string) -> str:
   return ':'.join(string.split(':')[:-1])
 
-
 def get_so_image_basename(string) -> str:
   return get_image_name(string).split('/so-')[-1]
 
-
 def get_image_version(string) -> str:
   ver = string.split(':')[-1]
   if ver == 'latest':
@@ -35,56 +29,75 @@ def get_image_version(string) -> str:
         return '999999.9.9'
     return ver
 
+def run_command(command):
+    process = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    if process.returncode != 0:
+        print(f"Error executing command: {command}", file=sys.stderr)
+        print(f"Error message: {process.stderr}", file=sys.stderr)
+        exit(1)
+    return process.stdout
 
 def main(quiet):
-  client = docker.from_env()
-
-  # Prune old/stopped containers
-  if not quiet: print('Pruning old containers')
-  client.containers.prune()
-
-  image_list = client.images.list(filters={ 'dangling': False })
-
-  # Map list of image objects to flattened list of tags (format: "name:version")
-  tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list))))
-
-  # Filter to only SO images (base name begins with "so-")
-  tag_list = list(filter(lambda x: re.match(r'^.*\/so-[^\/]*$', get_image_name(x)), tag_list))
-
-  # Group tags into lists by base name (sort by same projection first)
-  tag_list.sort(key=lambda x: get_so_image_basename(x))
-  grouped_tag_lists = [ list(it) for _, it in groupby(tag_list, lambda x: get_so_image_basename(x)) ]
-
-  no_prunable = True
-  for t_list in grouped_tag_lists:
     try:
-      # Group tags by version, in case multiple images exist with the same version string
-      t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
-      grouped_t_list = [ list(it) for _,it in groupby(t_list, lambda x: get_image_version(x)) ]
-
-      # Keep the 2 most current version groups
-      if len(grouped_t_list) <= 2:
-        continue
-      else:
-        no_prunable = False
-        for group in grouped_t_list[2:]:
-          for tag in group:
-            if not quiet: print(f'Removing image {tag}')
+        # Prune old/stopped containers using docker CLI
+        if not quiet: print('Pruning old containers')
+        run_command('docker container prune -f')
+        
+        # Get list of images using docker CLI
+        images_json = run_command('docker images --format "{{json .}}"')
+        
+        # Parse the JSON output
+        image_list = []
+        for line in images_json.strip().split('\n'):
+            if line:  # Skip empty lines
+                image_list.append(json.loads(line))
+        
+        # Extract tags in the format "name:version"
+        tag_list = []
+        for img in image_list:
+            # Skip dangling images
+            if img.get('Repository') != "<none>" and img.get('Tag') != "<none>":
+                tag = f"{img.get('Repository')}:{img.get('Tag')}"
+                # Filter to only SO images (base name begins with "so-")
+                if re.match(r'^.*\/so-[^\/]*$', get_image_name(tag)):
+                    tag_list.append(tag)
+        
+        # Group tags into lists by base name (sort by same projection first)
+        tag_list.sort(key=lambda x: get_so_image_basename(x))
+        grouped_tag_lists = [list(it) for k, it in groupby(tag_list, lambda x: get_so_image_basename(x))]
+        
+        no_prunable = True
+        for t_list in grouped_tag_lists:
             try:
-              client.images.remove(tag, force=True)
-            except docker.errors.ClientError as e:
-              print(f'Could not remove image {tag}, continuing...')
-    except (docker.errors.APIError, InvalidVersion) as e:
-      print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
-      exit(1)
+                # Group tags by version, in case multiple images exist with the same version string
+                t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
+                grouped_t_list = [list(it) for k, it in groupby(t_list, lambda x: get_image_version(x))]
+                # Keep the 2 most current version groups
+                if len(grouped_t_list) <= 2:
+                    continue
+                else:
+                    no_prunable = False
+                    for group in grouped_t_list[2:]:
+                        for tag in group:
+                            if not quiet: print(f'Removing image {tag}')
+                            try:
+                                run_command(f'docker rmi -f {tag}')
+                            except Exception as e:
+                                print(f'Could not remove image {tag}, continuing...')
+            except (InvalidVersion) as e:
+                print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
+                exit(1)
+            except Exception as e:
+                print('Unhandled exception occurred:')
+                print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
+                exit(1)
+        
+        if no_prunable and not quiet:
+            print('No Security Onion images to prune')
+    
     except Exception as e:
-      print('Unhandled exception occurred:')
-      print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
-      exit(1)
-
-  if no_prunable and not quiet:
-    print('No Security Onion images to prune')
-
+        print(f"Error: {e}", file=sys.stderr)
+        exit(1)
 
 if __name__ == "__main__":
   main_parser = argparse.ArgumentParser(add_help=False)

salt/common/tools/sbin/so-image-common

@@ -25,7 +25,6 @@ container_list() {
   if [ $MANAGERCHECK == 'so-import' ]; then
     TRUSTED_CONTAINERS=(
       "so-elasticsearch"
-      "so-idstools"
       "so-influxdb"
       "so-kibana"
       "so-kratos"
@@ -33,7 +32,6 @@ container_list() {
       "so-nginx"
       "so-pcaptools"
       "so-soc"
-      "so-steno"
       "so-suricata"
       "so-telegraf"
       "so-zeek"
@@ -49,7 +47,6 @@ container_list() {
       "so-elastic-fleet-package-registry"
       "so-elasticsearch"
       "so-idh"
-      "so-idstools"
       "so-influxdb"
       "so-kafka"
       "so-kibana"
@@ -60,10 +57,7 @@ container_list() {
       "so-pcaptools"
       "so-redis"
       "so-soc"
-      "so-steno"
       "so-strelka-backend"
-      "so-strelka-filestream"
-      "so-strelka-frontend"
       "so-strelka-manager"
       "so-suricata"
       "so-telegraf"
@@ -71,12 +65,10 @@ container_list() {
     )
   else
     TRUSTED_CONTAINERS=(
-      "so-idstools"
       "so-elasticsearch"
       "so-logstash"
       "so-nginx"
       "so-redis"
-      "so-steno"
       "so-suricata"
       "so-soc"
       "so-telegraf"

salt/common/tools/sbin/so-log-check

@@ -128,6 +128,9 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No shard available"           # Typical error when making a query before ES has finished loading all indices
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|responded with status-code 503" # telegraf getting 503 from ES during startup
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process_cluster_event_timeout_exception" # logstash waiting for elasticsearch to start
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not configured for GeoIP"     # SO does not bundle the maxminddb with Zeek
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|HTTP 404: Not Found"          # Salt loops until Kratos returns 200, during startup Kratos may not be ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Cancelling deferred write event maybeFenceReplicas because the event queue is now closed" # Kafka controller log during shutdown/restart
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -157,6 +160,10 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding index lifecycle policy" # false positive (elasticsearch policy names contain 'error') 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding ingest pipeline"       # false positive (elasticsearch ingest pipeline names contain 'error') 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating index template"      # false positive (elasticsearch index or template names contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating component template"  # false positive (elasticsearch index or template names contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading component template"  # false positive (elasticsearch index or template names contain 'error')
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error while parsing document for index \[.ds-logs-kratos-so-.*object mapping for \[file\]" # false positive (mapping error occuring BEFORE kratos index has rolled over in 2.4.210)
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -172,7 +179,6 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|salt-minion-check"            # bug in early 2.4 place Jinja script in non-jinja salt dir causing cron output errors
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|monitoring.metrics"           # known issue with elastic agent casting the field incorrectly if an integer value shows up before a float
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|repodownload.conf"            # known issue with reposync on pre-2.4.20 
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing versions record"      # stenographer corrupt index
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|soc.field."                   # known ingest type collisions issue with earlier versions of SO
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error parsing signature"      # Malformed Suricata rule, from upstream provider
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sticky buffer has no matches" # Non-critical Suricata error
@@ -219,6 +225,9 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Initialized license manager"  # SOC log: before fields.status was changed to fields.licenseStatus
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tcp 127.0.0.1:6791: bind: address already in use" # so-elastic-fleet agent restarting. Seen starting w/ 8.18.8 https://github.com/elastic/kibana/issues/201459
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|TransformTask\] \[logs-(tychon|aws_billing|microsoft_defender_endpoint).*user so_kibana lacks the required permissions \[logs-\1" # Known issue with 3 integrations using kibana_system role vs creating unique api creds with proper permissions.
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|manifest unknown"             # appears in so-dockerregistry log for so-tcpreplay following docker upgrade to 29.2.1-1
 fi
 
 RESULT=0
@@ -265,6 +274,13 @@ for log_file in $(cat /tmp/log_check_files); do
     tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check
     check_for_errors
 done
+# Look for OOM specific errors in /var/log/messages which can lead to odd behavior / test failures
+if [[ -f /var/log/messages ]]; then
+    status "Checking log file /var/log/messages"
+    if journalctl --since "24 hours ago" | grep -iE 'out of memory|oom-kill'; then
+        RESULT=1
+    fi
+fi
 
 # Cleanup temp files
 rm -f /tmp/log_check_files

salt/common/tools/sbin/so-nsm-clear

@@ -55,19 +55,22 @@ if [ $SKIP -ne 1 ]; then
 fi
 
 delete_pcap() {
-  PCAP_DATA="/nsm/pcap/"
-  [ -d $PCAP_DATA ] && so-pcap-stop && rm -rf $PCAP_DATA/* && so-pcap-start
+  PCAP_DATA="/nsm/suripcap/"
+  [ -d $PCAP_DATA ] && rm -rf $PCAP_DATA/*
 }
 delete_suricata() {
   SURI_LOG="/nsm/suricata/"
-  [ -d $SURI_LOG ] && so-suricata-stop && rm -rf $SURI_LOG/* && so-suricata-start
+  [ -d $SURI_LOG ] && rm -rf $SURI_LOG/*
 }
 delete_zeek() {
   ZEEK_LOG="/nsm/zeek/logs/"
   [ -d $ZEEK_LOG ] && so-zeek-stop && rm -rf $ZEEK_LOG/* && so-zeek-start
 }
 
+so-suricata-stop
 delete_pcap
 delete_suricata
 delete_zeek
+so-suricata-start
+
 

salt/common/tools/sbin/so-restart

@@ -23,7 +23,6 @@ if [ $# -ge 1 ]; then
         fi
 
         case $1 in
-                "steno") docker stop so-steno && docker rm so-steno && salt-call state.apply pcap queue=True;;
                 "elastic-fleet") docker stop so-elastic-fleet && docker rm so-elastic-fleet && salt-call state.apply elasticfleet queue=True;;
                 *)  docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
         esac

salt/common/tools/sbin/so-sensor-clean

@@ -72,7 +72,7 @@ clean() {
 		done
 	fi
 
-	## Clean up extracted pcaps from Steno
+	## Clean up extracted pcaps
 	PCAPS='/nsm/pcapout'
 	OLDEST_PCAP=$(find $PCAPS -type f -printf '%T+ %p\n' | sort -n | head -n 1)
 	if [ -z "$OLDEST_PCAP" -o "$OLDEST_PCAP" == ".." -o "$OLDEST_PCAP" == "." ]; then

salt/common/tools/sbin/so-start

@@ -23,7 +23,6 @@ if [ $# -ge 1 ]; then
 
 	case $1 in
    		"all") salt-call state.highstate queue=True;;
-   		"steno") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply pcap queue=True; fi ;; 
    		"elastic-fleet") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply elasticfleet queue=True; fi ;; 
    		*) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; 
 	esac

salt/common/tools/sbin/so_logging_utils.py

@@ -0,0 +1,53 @@
+#!/usr/bin/python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import logging
+import os 
+import sys
+
+def setup_logging(logger_name, log_file_path, log_level=logging.INFO, format_str='%(asctime)s - %(levelname)s - %(message)s'):
+    """
+    Sets up logging for a script.
+
+    Parameters:
+        logger_name (str): The name of the logger.
+        log_file_path (str): The file path for the log file.
+        log_level (int): The logging level (e.g., logging.INFO, logging.DEBUG).
+        format_str (str): The format string for log messages.
+
+    Returns:
+        logging.Logger: Configured logger object.
+    """
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(log_level)
+
+    # Create directory for log file if it doesn't exist
+    log_file_dir = os.path.dirname(log_file_path)
+    if log_file_dir and not os.path.exists(log_file_dir):
+        try:
+            os.makedirs(log_file_dir)
+        except OSError as e:
+            print(f"Error creating directory {log_file_dir}: {e}")
+            sys.exit(1)
+
+    # Create handlers
+    c_handler = logging.StreamHandler()
+    f_handler = logging.FileHandler(log_file_path)
+    c_handler.setLevel(log_level)
+    f_handler.setLevel(log_level)
+
+    # Create formatter and add it to handlers
+    formatter = logging.Formatter(format_str)
+    c_handler.setFormatter(formatter)
+    f_handler.setFormatter(formatter)
+
+    # Add handlers to the logger if they are not already added
+    if not logger.hasHandlers():
+        logger.addHandler(c_handler)
+        logger.addHandler(f_handler)
+
+    return logger

salt/common/tools/sbin_jinja/so-desktop-install

@@ -6,7 +6,7 @@
 # Elastic License 2.0.
 
 source /usr/sbin/so-common
-doc_desktop_url="$DOC_BASE_URL/desktop.html"
+doc_desktop_url="$DOC_BASE_URL/desktop"
 
 {# we only want the script to install the desktop if it is OEL -#}
 {% if grains.os == 'OEL' -%}

salt/common/tools/sbin_jinja/so-import-pcap

@@ -85,7 +85,7 @@ function suricata() {
   docker run --rm \
     -v /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro \
     -v /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro \
-    -v /opt/so/conf/suricata/rules:/etc/suricata/rules:ro \
+    -v /opt/so/rules/suricata/:/etc/suricata/rules:ro \
     -v ${LOG_PATH}:/var/log/suricata/:rw \
     -v ${NSM_PATH}/:/nsm/:rw \
     -v "$PCAP:/input.pcap:ro" \
@@ -173,7 +173,7 @@ for PCAP in $INPUT_FILES; do
   status "- assigning unique identifier to import: $HASH"
 
   pcap_data=$(pcapinfo "${PCAP}")
-  if ! echo "$pcap_data" | grep -q "First packet time:" || echo "$pcap_data" |egrep -q "Last packet time:    1970-01-01|Last packet time:    n/a"; then
+  if ! echo "$pcap_data" | grep -q "Earliest packet time:" || echo "$pcap_data" |egrep -q "Latest packet time:   1970-01-01|Latest packet time:   n/a"; then
     status "- this PCAP file is invalid; skipping"
     INVALID_PCAPS_COUNT=$((INVALID_PCAPS_COUNT + 1))
   else
@@ -205,8 +205,8 @@ for PCAP in $INPUT_FILES; do
       HASHES="${HASHES} ${HASH}"
     fi
 
-    START=$(pcapinfo "${PCAP}" -a |grep "First packet time:" | awk '{print $4}')
-    END=$(pcapinfo "${PCAP}" -e |grep "Last packet time:" | awk '{print $4}')
+    START=$(pcapinfo "${PCAP}" -a |grep "Earliest packet time:" | awk '{print $4}')
+    END=$(pcapinfo "${PCAP}" -e |grep "Latest packet time:" | awk '{print $4}')
     status "- found PCAP data spanning dates $START through $END"
 
     # compare $START to $START_OLDEST
@@ -248,7 +248,7 @@ fi
 START_OLDEST_SLASH=$(echo $START_OLDEST | sed -e 's/-/%2F/g')
 END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
 if [[ $VALID_PCAPS_COUNT -gt 0 ]] || [[ $SKIPPED_PCAPS_COUNT -gt 0 ]]; then
-  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20event.module*%20%7C%20groupby%20-sankey%20event.module*%20event.dataset%20%7C%20groupby%20event.dataset%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20network.protocol%20%7C%20groupby%20rule.name%20rule.category%20event.severity_label%20%7C%20groupby%20dns.query.name%20%7C%20groupby%20file.mime_type%20%7C%20groupby%20http.virtual_host%20http.uri%20%7C%20groupby%20notice.note%20notice.message%20notice.sub_message%20%7C%20groupby%20ssl.server_name%20%7C%20groupby%20source_geo.organization_name%20source.geo.country_name%20%7C%20groupby%20destination_geo.organization_name%20destination.geo.country_name&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
+  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20event.module*%20%7C%20groupby%20-sankey%20event.module*%20event.dataset%20%7C%20groupby%20event.dataset%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20network.protocol%20%7C%20groupby%20rule.name%20rule.category%20event.severity_label%20%7C%20groupby%20dns.query.name%20%7C%20groupby%20file.mime_type%20%7C%20groupby%20http.virtual_host%20http.uri%20%7C%20groupby%20notice.note%20notice.message%20notice.sub_message%20%7C%20groupby%20ssl.server_name%20%7C%20groupby%20source.as.organization.name%20source.geo.country_name%20%7C%20groupby%20destination.as.organization.name%20destination.geo.country_name&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
 
   status "Import complete!"
   status

salt/common/tools/sbin_jinja/so-salt-emit-vm-deployment-status-event

@@ -0,0 +1,132 @@
+#!/opt/saltstack/salt/bin/python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{%   if 'vrt' in salt['pillar.get']('features', []) -%}
+
+"""
+Script for emitting VM deployment status events to the Salt event bus.
+
+This script provides functionality to emit status events for VM deployment operations,
+used by various Security Onion VM management tools.
+
+Usage:
+    so-salt-emit-vm-deployment-status-event -v <vm_name> -H <hypervisor> -s <status>
+
+Arguments:
+    -v, --vm-name     Name of the VM (hostname_role)
+    -H, --hypervisor  Name of the hypervisor
+    -s, --status      Current deployment status of the VM
+
+Example:
+    so-salt-emit-vm-deployment-status-event -v sensor1_sensor -H hypervisor1 -s "Creating"
+"""
+
+import sys
+import argparse
+import logging
+import salt.client
+from typing import Dict, Any
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+log = logging.getLogger(__name__)
+
+def emit_event(vm_name: str, hypervisor: str, status: str) -> bool:
+    """
+    Emit a VM deployment status event to the salt event bus.
+    
+    Args:
+        vm_name: Name of the VM (hostname_role)
+        hypervisor: Name of the hypervisor
+        status: Current deployment status of the VM
+        
+    Returns:
+        bool: True if event was sent successfully, False otherwise
+    
+    Raises:
+        ValueError: If status is not a valid deployment status
+    """
+    log.info("Attempting to emit deployment event...")
+    
+    try:
+        caller = salt.client.Caller()
+        event_data = {
+            'vm_name': vm_name,
+            'hypervisor': hypervisor,
+            'status': status
+        }
+        
+        # Use consistent event tag structure
+        event_tag = f'soc/dyanno/hypervisor/{status.lower()}'
+        
+        ret = caller.cmd(
+            'event.send',
+            event_tag,
+            event_data
+        )
+        
+        if not ret:
+            log.error("Failed to emit VM deployment status event: %s", event_data)
+            return False
+            
+        log.info("Successfully emitted VM deployment status event: %s", event_data)
+        return True
+        
+    except Exception as e:
+        log.error("Error emitting VM deployment status event: %s", str(e))
+        return False
+
+def parse_args():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description='Emit VM deployment status events to the Salt event bus.'
+    )
+    parser.add_argument('-v', '--vm-name', required=True,
+                        help='Name of the VM (hostname_role)')
+    parser.add_argument('-H', '--hypervisor', required=True,
+                        help='Name of the hypervisor')
+    parser.add_argument('-s', '--status', required=True,
+                        help='Current deployment status of the VM')
+    return parser.parse_args()
+
+def main():
+    """Main entry point for the script."""
+    try:
+        args = parse_args()
+        
+        success = emit_event(
+            vm_name=args.vm_name,
+            hypervisor=args.hypervisor,
+            status=args.status
+        )
+        
+        if not success:
+            sys.exit(1)
+            
+    except Exception as e:
+        log.error("Failed to emit status event: %s", str(e))
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()
+
+{%- else -%}
+
+echo "Hypervisor nodes are a feature supported only for customers with a valid license. \
+      Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com \
+      for more information about purchasing a license to enable this feature."
+
+{% endif -%}

salt/desktop/trusted-ca.sls

@@ -3,29 +3,16 @@
 {# we only want this state to run it is CentOS #}
 {% if GLOBALS.os == 'OEL' %}
 
-  {% set global_ca_text = [] %}
-  {% set global_ca_server = [] %}
-  {% set manager = GLOBALS.manager %}
-  {% set x509dict = salt['mine.get'](manager | lower~'*', 'x509.get_pem_entries') %}
-    {% for host in x509dict %}
-      {% if host.split('_')|last in ['manager', 'managersearch', 'standalone', 'import', 'eval'] %}
-        {% do global_ca_text.append(x509dict[host].get('/etc/pki/ca.crt')|replace('\n', '')) %}
-        {% do global_ca_server.append(host) %}
-      {% endif %}
-    {% endfor %}
-  {% set trusttheca_text = global_ca_text[0] %}
-  {% set ca_server = global_ca_server[0] %}
-
 trusted_ca:
-  x509.pem_managed:
+  file.managed:
     - name: /etc/pki/ca-trust/source/anchors/ca.crt
-    - text:  {{ trusttheca_text }}
+    - source: salt://ca/files/ca.crt
 
 update_ca_certs:
   cmd.run:
     - name: update-ca-trust
     - onchanges:
-      - x509: trusted_ca
+      - file: trusted_ca
 
 {% else %}
 

salt/docker/defaults.yaml

@@ -24,11 +24,6 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
-    'so-idstools':
-      final_octet: 25
-      custom_bind_mounts: []
-      extra_hosts: []
-      extra_env: []
     'so-influxdb':
       final_octet: 26
       port_bindings:
@@ -179,11 +174,6 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
-    'so-steno':
-      final_octet: 99
-      custom_bind_mounts: []
-      extra_hosts: []
-      extra_env: []
     'so-suricata':
       final_octet: 99
       custom_bind_mounts: []
@@ -200,6 +190,7 @@ docker:
       final_octet: 88
       port_bindings:
         - 0.0.0.0:9092:9092
+        - 0.0.0.0:29092:29092
         - 0.0.0.0:9093:9093
         - 0.0.0.0:8778:8778
       custom_bind_mounts: []

salt/docker/init.sls

@@ -6,9 +6,9 @@
 {% from 'docker/docker.map.jinja' import DOCKER %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
-# include ssl since docker service requires the intca
+# docker service requires the ca.crt
 include:
-  - ssl
+  - ca
 
 dockergroup:
   group.present:
@@ -20,20 +20,20 @@ dockergroup:
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.7.21-1
-      - docker-ce: 5:27.2.0-1~debian.12~bookworm
-      - docker-ce-cli: 5:27.2.0-1~debian.12~bookworm
-      - docker-ce-rootless-extras: 5:27.2.0-1~debian.12~bookworm
+      - containerd.io: 2.2.1-1~debian.12~bookworm
+      - docker-ce: 5:29.2.1-1~debian.12~bookworm
+      - docker-ce-cli: 5:29.2.1-1~debian.12~bookworm
+      - docker-ce-rootless-extras: 5:29.2.1-1~debian.12~bookworm
     - hold: True
     - update_holds: True
 {%    elif grains.oscodename == 'jammy' %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.7.21-1
-      - docker-ce: 5:27.2.0-1~ubuntu.22.04~jammy
-      - docker-ce-cli: 5:27.2.0-1~ubuntu.22.04~jammy
-      - docker-ce-rootless-extras: 5:27.2.0-1~ubuntu.22.04~jammy
+      - containerd.io: 2.2.1-1~ubuntu.22.04~jammy
+      - docker-ce: 5:29.2.1-1~ubuntu.22.04~jammy
+      - docker-ce-cli: 5:29.2.1-1~ubuntu.22.04~jammy
+      - docker-ce-rootless-extras: 5:29.2.1-1~ubuntu.22.04~jammy
     - hold: True
     - update_holds: True
 {%    else %}
@@ -51,10 +51,10 @@ dockerheldpackages:
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.7.21-3.1.el9
-      - docker-ce: 3:27.2.0-1.el9
-      - docker-ce-cli: 1:27.2.0-1.el9
-      - docker-ce-rootless-extras: 27.2.0-1.el9
+      - containerd.io: 2.2.1-1.el9
+      - docker-ce: 3:29.2.1-1.el9
+      - docker-ce-cli: 1:29.2.1-1.el9
+      - docker-ce-rootless-extras: 29.2.1-1.el9
     - hold: True
     - update_holds: True
 {% endif %}
@@ -89,10 +89,9 @@ docker_running:
     - enable: True
     - watch:
       - file: docker_daemon
-      - x509: trusttheca
     - require:
       - file: docker_daemon
-      - x509: trusttheca
+      - file: trusttheca
 
 
 # Reserve OS ports for Docker proxy in case boot settings are not already applied/present
@@ -118,4 +117,4 @@ sos_docker_net:
         com.docker.network.bridge.enable_ip_masquerade: 'true'
         com.docker.network.bridge.enable_icc: 'true'
         com.docker.network.bridge.host_binding_ipv4: '0.0.0.0'
-    - unless: 'docker network ls | grep sobridge'
+    - unless: ip l | grep sobridge

salt/docker/soc_docker.yaml

@@ -41,7 +41,6 @@ docker:
         forcedType: "[]string"
     so-elastic-fleet: *dockerOptions
     so-elasticsearch: *dockerOptions
-    so-idstools: *dockerOptions
     so-influxdb: *dockerOptions
     so-kibana: *dockerOptions
     so-kratos: *dockerOptions
@@ -63,7 +62,6 @@ docker:
     so-idh: *dockerOptions
     so-elastic-agent: *dockerOptions
     so-telegraf: *dockerOptions
-    so-steno: *dockerOptions
     so-suricata:
       final_octet:
         description: Last octet of the container IP address.
@@ -102,4 +100,4 @@ docker:
         multiline: True
         forcedType: "[]string"
     so-zeek: *dockerOptions
-    so-kafka: *dockerOptions
+    so-kafka: *dockerOptions

salt/elastalert/enabled.sls

@@ -60,7 +60,7 @@ so-elastalert:
     - watch:
       - file: elastaconf
     - onlyif:
-      - "so-elasticsearch-query / | jq -r '.version.number[0:1]' | grep -q 8" {# only run this state if elasticsearch is version 8 #}
+      - "so-elasticsearch-query / | jq -r '.version.number[0:1]' | grep -q 9" {# only run this state if elasticsearch is version 9 #}
 
 delete_so-elastalert_so-status.disabled:
   file.uncomment:

salt/elasticagent/enabled.sls

@@ -9,6 +9,7 @@
 {%   from 'docker/docker.map.jinja' import DOCKER %}
 
 include:
+  - ca
   - elasticagent.config
   - elasticagent.sostatus
 
@@ -55,8 +56,10 @@ so-elastic-agent:
       {% endif %}
     - require:
       - file: create-elastic-agent-config
+      - file: trusttheca
     - watch:
       - file: create-elastic-agent-config
+      - file: trusttheca
 
 delete_so-elastic-agent_so-status.disabled:
   file.uncomment:

salt/elasticagent/files/elastic-agent.yml.jinja

@@ -3,7 +3,7 @@
 {%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
 
 id: aea1ba80-1065-11ee-a369-97538913b6a9
-revision: 1
+revision: 4
 outputs:
   default:
     type: elasticsearch
@@ -22,242 +22,133 @@ agent:
     metrics: false
   features: {}
 inputs:
-  - id: logfile-logs-fefef78c-422f-4cfa-8abf-4cd1b9428f62
-    name: import-evtx-logs
-    revision: 2
-    type: logfile
-    use_output: default
-    meta:
-      package:
-        name: log
-        version:
-    data_stream:
-      namespace: so
-    package_policy_id: fefef78c-422f-4cfa-8abf-4cd1b9428f62
-    streams:
-      - id: logfile-log.log-fefef78c-422f-4cfa-8abf-4cd1b9428f62
-        data_stream:
-          dataset: import
-        paths:
-          - /nsm/import/*/evtx/*.json
-        processors:
-          - dissect:
-              field: log.file.path
-              tokenizer: '/nsm/import/%{import.id}/evtx/%{import.file}'
-              target_prefix: ''
-          - decode_json_fields:
-              fields:
-                - message
-              target: ''
-          - drop_fields:
-              ignore_missing: true
-              fields:
-                - host
-          - add_fields:
-              fields:
-                dataset: system.security
-                type: logs
-                namespace: default
-              target: data_stream
-          - add_fields:
-              fields:
-                dataset: system.security
-                module: system
-                imported: true
-              target: event
-          - then:
-              - add_fields:
-                  fields:
-                    dataset: windows.sysmon_operational
-                  target: data_stream
-              - add_fields:
-                  fields:
-                    dataset: windows.sysmon_operational
-                    module: windows
-                    imported: true
-                  target: event
-            if:
-              equals:
-                winlog.channel: Microsoft-Windows-Sysmon/Operational
-          - then:
-              - add_fields:
-                  fields:
-                    dataset: system.application
-                  target: data_stream
-              - add_fields:
-                  fields:
-                    dataset: system.application
-                  target: event
-            if:
-              equals:
-                winlog.channel: Application
-          - then:
-              - add_fields:
-                  fields:
-                    dataset: system.system
-                  target: data_stream
-              - add_fields:
-                  fields:
-                    dataset: system.system
-                  target: event
-            if:
-              equals:
-                winlog.channel: System
-          - then:
-              - add_fields:
-                  fields:
-                    dataset: windows.powershell_operational
-                  target: data_stream
-              - add_fields:
-                  fields:
-                    dataset: windows.powershell_operational
-                    module: windows
-                  target: event
-            if:
-              equals:
-                winlog.channel: Microsoft-Windows-PowerShell/Operational
-        tags:
-          - import
-  - id: logfile-redis-fc98c947-7d17-4861-a318-7ad075f6d1b0
-    name: redis-logs
-    revision: 2
-    type: logfile
-    use_output: default
-    meta:
-      package:
-        name: redis
-        version:
-    data_stream:
-      namespace: default
-    package_policy_id: fc98c947-7d17-4861-a318-7ad075f6d1b0
-    streams:
-      - id: logfile-redis.log-fc98c947-7d17-4861-a318-7ad075f6d1b0
-        data_stream:
-          dataset: redis.log
-          type: logs
-        exclude_files:
-          - .gz$
-        paths:
-          - /opt/so/log/redis/redis.log
-        tags:
-          - redis-log
-        exclude_lines:
-          - '^\s+[\-`(''.|_]'
-  - id: logfile-logs-3b56803d-5ade-4c93-b25e-9b37182f66b8
+  - id: filestream-filestream-85820eb0-25ef-11f0-a18d-1b26f69b8310
     name: import-suricata-logs
-    revision: 2
-    type: logfile
+    revision: 3
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: 3b56803d-5ade-4c93-b25e-9b37182f66b8
+    package_policy_id: 85820eb0-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-3b56803d-5ade-4c93-b25e-9b37182f66b8
+      - id: filestream-filestream.generic-85820eb0-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: import
-        pipeline: suricata.common
         paths:
           - /nsm/import/*/suricata/eve*.json
+        pipeline: suricata.common
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - \.gz$
+        ignore_older: 72h
+        clean_inactive: -1
+        parsers: null
         processors:
           - add_fields:
+              target: event
               fields:
+                category: network
                 module: suricata
                 imported: true
-                category: network
-              target: event
           - dissect:
+              tokenizer: /nsm/import/%{import.id}/suricata/%{import.file}
               field: log.file.path
-              tokenizer: '/nsm/import/%{import.id}/suricata/%{import.file}'
               target_prefix: ''
-  - id: logfile-logs-c327e1a3-1ebe-449c-a8eb-f6f35032e69d
-    name: soc-server-logs
-    revision: 2
-    type: logfile
+        file_identity.native: null
+        prospector.scanner.fingerprint.enabled: false
+  - id: filestream-filestream-86b4e960-25ef-11f0-a18d-1b26f69b8310
+    name: import-zeek-logs
+    revision: 3
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+    package_policy_id: 86b4e960-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-c327e1a3-1ebe-449c-a8eb-f6f35032e69d
+      - id: filestream-filestream.generic-86b4e960-25ef-11f0-a18d-1b26f69b8310
         data_stream:
-          dataset: soc
-        pipeline: common
+          dataset: import
         paths:
-          - /opt/so/log/soc/sensoroni-server.log
+          - /nsm/import/*/zeek/logs/*.log
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - >-
+            (broker|capture_loss|cluster|conn-summary|console|ecat_arp_info|known_certs|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout).log$
+        clean_inactive: -1
+        parsers: null
         processors:
-          - decode_json_fields:
-              add_error_key: true
-              process_array: true
-              max_depth: 2
-              fields:
-                - message
-              target: soc
+          - dissect:
+              tokenizer: /nsm/import/%{import.id}/zeek/logs/%{import.file}
+              field: log.file.path
+              target_prefix: ''
+          - script:
+              lang: javascript
+              source: |
+                function process(event) {
+                  var pl = event.Get("import.file").slice(0,-4);
+                  event.Put("@metadata.pipeline", "zeek." + pl);
+                }
           - add_fields:
-              fields:
-                module: soc
-                dataset_temp: server
-                category: host
               target: event
-          - rename:
-              ignore_missing: true
               fields:
-                - from: soc.fields.sourceIp
-                  to: source.ip
-                - from: soc.fields.status
-                  to: http.response.status_code
-                - from: soc.fields.method
-                  to: http.request.method
-                - from: soc.fields.path
-                  to: url.path
-                - from: soc.message
-                  to: event.action
-                - from: soc.level
-                  to: log.level
-        tags:
-          - so-soc
-  - id: logfile-logs-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+                category: network
+                module: zeek
+                imported: true
+          - add_tags:
+              tags: ics
+              when:
+                regexp:
+                  import.file: >-
+                    ^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*
+        file_identity.native: null
+        prospector.scanner.fingerprint.enabled: false
+  - id: filestream-filestream-91741240-25ef-11f0-a18d-1b26f69b8310
     name: soc-sensoroni-logs
-    revision: 2
-    type: logfile
+    revision: 3
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: 906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+    package_policy_id: 91741240-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-906e0d4c-9ec3-4c6a-bef6-e347ec9fd073
+      - id: filestream-filestream.generic-91741240-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: soc
-        pipeline: common
         paths:
           - /opt/so/log/sensoroni/sensoroni.log
+        pipeline: common
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - \.gz$
+        clean_inactive: -1
+        parsers: null
         processors:
           - decode_json_fields:
-              add_error_key: true
-              process_array: true
-              max_depth: 2
               fields:
                 - message
               target: sensoroni
+              process_array: true
+              max_depth: 2
+              add_error_key: true
           - add_fields:
+              target: event
               fields:
+                category: host
                 module: soc
                 dataset_temp: sensoroni
-                category: host
-              target: event
           - rename:
-              ignore_missing: true
               fields:
                 - from: sensoroni.fields.sourceIp
                   to: source.ip
@@ -271,141 +162,100 @@ inputs:
                   to: event.action
                 - from: sensoroni.level
                   to: log.level
-  - id: logfile-logs-df0d7f2c-221f-433b-b18b-d1cf83250515
-    name: soc-salt-relay-logs
-    revision: 2
-    type: logfile
-    use_output: default
-    meta:
-      package:
-        name: log
-        version:
-    data_stream:
-      namespace: so
-    package_policy_id: df0d7f2c-221f-433b-b18b-d1cf83250515
-    streams:
-      - id: logfile-log.log-df0d7f2c-221f-433b-b18b-d1cf83250515
-        data_stream:
-          dataset: soc
-        pipeline: common
-        paths:
-          - /opt/so/log/soc/salt-relay.log
-        processors:
-          - dissect:
-              field: message
-              tokenizer: '%{soc.ts} | %{event.action}'
-              target_prefix: ''
-          - add_fields:
-              fields:
-                module: soc
-                dataset_temp: salt_relay
-                category: host
-              target: event
-        tags:
-          - so-soc
-  - id: logfile-logs-74bd2366-fe52-493c-bddc-843a017fc4d0
-    name: soc-auth-sync-logs
-    revision: 2
-    type: logfile
-    use_output: default
-    meta:
-      package:
-        name: log
-        version:
-    data_stream:
-      namespace: so
-    package_policy_id: 74bd2366-fe52-493c-bddc-843a017fc4d0
-    streams:
-      - id: logfile-log.log-74bd2366-fe52-493c-bddc-843a017fc4d0
-        data_stream:
-          dataset: soc
-        pipeline: common
-        paths:
-          - /opt/so/log/soc/sync.log
-        processors:
-          - dissect:
-              field: message
-              tokenizer: '%{event.action}'
-              target_prefix: ''
-          - add_fields:
-              fields:
-                module: soc
-                dataset_temp: auth_sync
-                category: host
-              target: event
-        tags:
-          - so-soc
-  - id: logfile-logs-d151d9bf-ff2a-4529-9520-c99244bc0253
+              ignore_missing: true
+        file_identity.native: null
+        prospector.scanner.fingerprint.enabled: false
+  - id: filestream-filestream-976e3900-25ef-11f0-a18d-1b26f69b8310
     name: suricata-logs
-    revision: 2
-    type: logfile
+    revision: 3
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: d151d9bf-ff2a-4529-9520-c99244bc0253
+    package_policy_id: 976e3900-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-d151d9bf-ff2a-4529-9520-c99244bc0253
+      - id: filestream-filestream.generic-976e3900-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: suricata
-        pipeline: suricata.common
         paths:
           - /nsm/suricata/eve*.json
+        pipeline: suricata.common
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - \.gz$
+        clean_inactive: -1
+        parsers: null
         processors:
           - add_fields:
-              fields:
-                module: suricata
-                category: network
               target: event
-  - id: logfile-logs-31f94d05-ae75-40ee-b9c5-0e0356eff327
+              fields:
+                category: network
+                module: suricata
+        file_identity.native: null
+        prospector.scanner.fingerprint.enabled: false
+  - id: filestream-filestream-95091fe0-25ef-11f0-a18d-1b26f69b8310
     name: strelka-logs
-    revision: 2
-    type: logfile
+    revision: 3
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
+        name: filestream
         version:
     data_stream:
       namespace: so
-    package_policy_id: 31f94d05-ae75-40ee-b9c5-0e0356eff327
+    package_policy_id: 95091fe0-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-31f94d05-ae75-40ee-b9c5-0e0356eff327
+      - id: filestream-filestream.generic-95091fe0-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: strelka
-        pipeline: strelka.file
         paths:
           - /nsm/strelka/log/strelka.log
+        pipeline: strelka.file
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - \.gz$
+        clean_inactive: -1
+        parsers: null
         processors:
           - add_fields:
-              fields:
-                module: strelka
-                category: file
               target: event
-  - id: logfile-logs-6197fe84-9b58-4d9b-8464-3d517f28808d
+              fields:
+                category: file
+                module: strelka
+        file_identity.native: null
+        prospector.scanner.fingerprint.enabled: false
+  - id: filestream-filestream-9f309ca0-25ef-11f0-a18d-1b26f69b8310
     name: zeek-logs
-    revision: 1
-    type: logfile
+    revision: 2
+    type: filestream
     use_output: default
     meta:
       package:
-        name: log
-        version: 
+        name: filestream
+        version:
     data_stream:
       namespace: so
-    package_policy_id: 6197fe84-9b58-4d9b-8464-3d517f28808d
+    package_policy_id: 9f309ca0-25ef-11f0-a18d-1b26f69b8310
     streams:
-      - id: logfile-log.log-6197fe84-9b58-4d9b-8464-3d517f28808d
+      - id: filestream-filestream.generic-9f309ca0-25ef-11f0-a18d-1b26f69b8310
         data_stream:
           dataset: zeek
         paths:
           - /nsm/zeek/logs/current/*.log
+        prospector.scanner.recursive_glob: true
+        prospector.scanner.exclude_files:
+          - >-
+            (broker|capture_loss|cluster|conn-summary|console|ecat_arp_info|known_certs|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout).log$
+        clean_inactive: -1
+        parsers: null
         processors:
           - dissect:
-              tokenizer: '/nsm/zeek/logs/current/%{pipeline}.log'
+              tokenizer: /nsm/zeek/logs/current/%{pipeline}.log
               field: log.file.path
               trim_chars: .log
               target_prefix: ''
@@ -427,18 +277,17 @@ inputs:
                 regexp:
                   pipeline: >-
                     ^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*
-        exclude_files:
-          - >-
-            broker|capture_loss|cluster|ecat_arp_info|known_hosts|known_services|loaded_scripts|ntp|ocsp|packet_filter|reporter|stats|stderr|stdout.log$
+        file_identity.native: null
+        prospector.scanner.fingerprint.enabled: false
   - id: udp-udp-35051de0-46a5-11ee-8d5d-9f98c8182f60
     name: syslog-udp-514
-    revision: 3
+    revision: 4
     type: udp
     use_output: default
     meta:
       package:
         name: udp
-        version: 1.10.0
+        version:
     data_stream:
       namespace: so
     package_policy_id: 35051de0-46a5-11ee-8d5d-9f98c8182f60
@@ -458,13 +307,13 @@ inputs:
           - syslog
   - id: tcp-tcp-33d37bb0-46a5-11ee-8d5d-9f98c8182f60
     name: syslog-tcp-514
-    revision: 3
+    revision: 4
     type: tcp
     use_output: default
     meta:
       package:
         name: tcp
-        version: 1.10.0
+        version:
     data_stream:
       namespace: so
     package_policy_id: 33d37bb0-46a5-11ee-8d5d-9f98c8182f60

salt/elasticfleet/artifact_registry.sls

@@ -9,3 +9,6 @@ fleetartifactdir:
     - user: 947
     - group: 939
     - makedirs: True
+    - recurse:
+      - user
+      - group

salt/elasticfleet/config.map.jinja

@@ -0,0 +1,34 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
+
+{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+
+{# advanced config_yaml options for elasticfleet logstash output #}
+{% set ADV_OUTPUT_LOGSTASH_RAW = ELASTICFLEETMERGED.config.outputs.logstash %}
+{% set ADV_OUTPUT_LOGSTASH = {} %}
+{% for k, v in ADV_OUTPUT_LOGSTASH_RAW.items() %}
+{%   if v != "" and v is not none %}
+{%     if k == 'queue_mem_events' %}
+{#       rename queue_mem_events queue.mem.events #}
+{%       do ADV_OUTPUT_LOGSTASH.update({'queue.mem.events':v}) %}
+{%     elif k == 'loadbalance' %}
+{%       if v %}
+{#         only include loadbalance config when its True #}
+{%         do ADV_OUTPUT_LOGSTASH.update({k:v}) %}
+{%       endif %}
+{%     else %}
+{%       do ADV_OUTPUT_LOGSTASH.update({k:v}) %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+{% set LOGSTASH_CONFIG_YAML_RAW = [] %}
+{% if ADV_OUTPUT_LOGSTASH %}
+{%   for k, v in ADV_OUTPUT_LOGSTASH.items() %}
+{%     do LOGSTASH_CONFIG_YAML_RAW.append(k ~ ': ' ~ v) %}
+{%   endfor %}
+{% endif %}
+
+{% set LOGSTASH_CONFIG_YAML = LOGSTASH_CONFIG_YAML_RAW | join('\\n') if LOGSTASH_CONFIG_YAML_RAW else '' %}

salt/elasticfleet/config.sls

@@ -9,6 +9,10 @@
 {% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
 {% set node_data = salt['pillar.get']('node_data') %}
 
+include:
+  - elasticfleet.artifact_registry
+  - elasticfleet.ssl
+
 # Add EA Group
 elasticfleetgroup:
   group.present:
@@ -92,6 +96,9 @@ soresourcesrepoclone:
     - rev: 'main'
     - depth: 1
     - force_reset: True
+    - retry:
+        attempts: 3
+        interval: 10
 {% endif %}
 
 elasticdefendconfdir:
@@ -166,7 +173,7 @@ eaoptionalintegrationsdir:
 
 {% for minion in node_data %}
 {% set role = node_data[minion]["role"] %}
-{% if role in [ "eval","fleet","heavynode","import","manager","managersearch","standalone" ] %}
+{% if role in [ "eval","fleet","heavynode","import","manager", "managerhype", "managersearch","standalone" ] %}
 {% set optional_integrations = ELASTICFLEETMERGED.optional_integrations %}
 {% set integration_keys = optional_integrations.keys() %}
 fleet_server_integrations_{{ minion }}:

salt/elasticfleet/defaults.yaml

@@ -10,11 +10,19 @@ elasticfleet:
       grid_enrollment: ''
     defend_filters:
       enable_auto_configuration: False
+    outputs:
+      logstash:
+        bulk_max_size: ''
+        worker: ''
+        queue_mem_events: ''
+        timeout: ''
+        loadbalance: False
+        compression_level: ''
     subscription_integrations: False
+    auto_upgrade_integrations: False
   logging:
     zeek:
       excluded:
-        - analyzer
         - broker
         - capture_loss
         - cluster
@@ -37,6 +45,7 @@ elasticfleet:
     - elasticsearch
     - endpoint
     - fleet_server
+    - filestream
     - http_endpoint
     - httpjson
     - log

salt/elasticfleet/enabled.sls

@@ -13,9 +13,10 @@
 {%   set SERVICETOKEN = salt['pillar.get']('elasticfleet:config:server:es_token','') %}
 
 include:
+  - ca
+  - logstash.ssl
   - elasticfleet.config
   - elasticfleet.sostatus
-  - ssl
 
 {% if grains.role not in ['so-fleet'] %}
 # Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready
@@ -32,6 +33,17 @@ so-elastic-fleet-auto-configure-logstash-outputs:
     - retry:
         attempts: 4
         interval: 30
+
+{# Separate from above in order to catch elasticfleet-logstash.crt changes and force update to fleet output policy #}
+so-elastic-fleet-auto-configure-logstash-outputs-force:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-outputs-update --certs
+    - retry:
+        attempts: 4
+        interval: 30
+    - onchanges:
+        - x509: etc_elasticfleet_logstash_crt
+        - x509: elasticfleet_kafka_crt
 {% endif %}
 
 # If enabled, automatically update Fleet Server URLs & ES Connection
@@ -67,6 +79,8 @@ so-elastic-fleet-auto-configure-artifact-urls:
 elasticagent_syncartifacts:
   file.recurse:
     - name: /nsm/elastic-fleet/artifacts/beats
+    - user: 947
+    - group: 947
     - source: salt://beats
 {% endif %}
 
@@ -120,6 +134,11 @@ so-elastic-fleet:
         {% endfor %}
       {% endif %}
     - watch:
+      - file: trusttheca
+      - x509: etc_elasticfleet_key
+      - x509: etc_elasticfleet_crt
+    - require:
+      - file: trusttheca
       - x509: etc_elasticfleet_key
       - x509: etc_elasticfleet_crt
 {%   endif %}
@@ -133,12 +152,18 @@ so-elastic-fleet-package-statefile:
 so-elastic-fleet-package-upgrade:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
     - onchanges:
       - file: /opt/so/state/elastic_fleet_packages.txt
 
 so-elastic-fleet-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-integration-policy-load
+    - retry:
+        attempts: 3
+        interval: 10
 
 so-elastic-agent-grid-upgrade:
   cmd.run:
@@ -150,7 +175,11 @@ so-elastic-agent-grid-upgrade:
 so-elastic-fleet-integration-upgrade:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-integration-upgrade
+    - retry:
+        attempts: 3
+        interval: 10
 
+{# Optional integrations script doesn't need the retries like so-elastic-fleet-integration-upgrade which loads the default integrations #}
 so-elastic-fleet-addon-integrations:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-optional-integrations-load

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/import-zeek-logs.json

@@ -2,7 +2,7 @@
 {%- raw -%}
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "import-zeek-logs",
@@ -10,19 +10,31 @@
   "description": "Zeek Import logs",
   "policy_id": "so-grid-nodes_general",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/zeek/logs/*.log"
             ],
             "data_stream.dataset": "import",
-            "tags": [],
+            "pipeline": "",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
+            "include_files": [],
             "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"import.file\").slice(0,-4);\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n      imported: true\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
-            "custom": "exclude_files: [\"{%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}.log$\"]\n"
+            "tags": [],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json

@@ -0,0 +1,61 @@
+{%- set identities = salt['sqlite3.fetch']('/nsm/kratos/db/db.sqlite', 'SELECT id, json_extract(traits, "$.email") as email FROM identities;') -%}
+{%- set valid_identities = false -%}
+{%- if identities -%}
+  {%- set valid_identities = true -%}
+  {%- for id, email in identities -%}
+    {%- if not id or not email -%}
+      {%- set valid_identities = false -%}
+      {%- break -%}
+    {%- endif -%}
+  {%- endfor -%}
+{%- endif -%}
+{
+  "package": {
+    "name": "filestream",
+    "version": ""
+  },
+  "name": "kratos-logs",
+  "description": "Kratos logs",
+  "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
+  "inputs": {
+    "filestream-filestream": {
+      "enabled": true,
+      "streams": {
+        "filestream.generic": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/kratos/kratos.log"
+            ],
+            "data_stream.dataset": "kratos",
+            "pipeline": "kratos",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
+            {%- if valid_identities -%}
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos\n- if:\n    has_fields:\n      - identity_id\n  then:{% for id, email in identities %}\n    - if:\n        equals:\n          identity_id: \"{{ id }}\"\n      then:\n        - add_fields:\n            target: ''\n            fields:\n              user.name: \"{{ email }}\"{% endfor %}",
+            {%- else -%}
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos",
+            {%- endif -%}
+            "tags": [
+              "so-kratos"
+            ],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
+          }
+        }
+      }
+    }
+  },
+  "force": true
+}

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/zeek-logs.json

@@ -2,28 +2,38 @@
 {%- raw -%}
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
-  "id": "zeek-logs",
   "name": "zeek-logs",
   "namespace": "so",
   "description": "Zeek logs",
   "policy_id": "so-grid-nodes_general",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/zeek/logs/current/*.log"
             ],
             "data_stream.dataset": "zeek",
-            "tags": [],
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": ["({%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%})(\\..+)?\\.log$"],
+            "include_files": [],
             "processors": "- dissect:\n    tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n    field: \"log.file.path\"\n    trim_chars: \".log\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\");\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"",
-            "custom": "exclude_files: [\"{%- endraw -%}{{ ELASTICFLEETMERGED.logging.zeek.excluded | join('|') }}{%- raw -%}.log$\"]\n"
+            "tags": [],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
@@ -31,4 +41,4 @@
   },
   "force": true
 }
-{%- endraw -%}
+{%- endraw -%}

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -1,32 +1,33 @@
 {
-	"name": "elastic-defend-endpoints",
-	"namespace": "default",
-	"description": "",
-	"package": {
-	  "name": "endpoint",
-	  "title": "Elastic Defend",
-	  "version": "8.17.0",
-	  "requires_root": true
-	},
-	"enabled": true,
-	"policy_id": "endpoints-initial",
-	"vars": {},
-	"inputs": [
-	  {
-		"type": "endpoint",
-		"enabled": true,
-		"config": {
-		  "integration_config": {
-			"value": {
-			  "type": "endpoint",
-			  "endpointConfig": {
-				"preset": "DataCollection"
-			  }
-			}
-		  }
-		},
-		"streams": []
-	  }
-	]
-  }
-  
+  "name": "elastic-defend-endpoints",
+  "namespace": "default",
+  "description": "",
+  "package": {
+    "name": "endpoint",
+    "title": "Elastic Defend",
+    "version": "9.0.2",
+    "requires_root": true
+  },
+  "enabled": true,
+  "policy_ids": [
+    "endpoints-initial"
+  ],
+  "vars": {},
+  "inputs": [
+    {
+      "type": "ENDPOINT_INTEGRATION_CONFIG",
+      "enabled": true,
+      "config": {
+        "_config": {
+          "value": {
+            "type": "endpoint",
+            "endpointConfig": {
+              "preset": "DataCollection"
+            }
+          }
+        }
+      },
+      "streams": []
+    }
+  ]
+}

salt/elasticfleet/files/integrations/grid-nodes_general/elastic-agent-monitor.json

@@ -0,0 +1,48 @@
+{
+  "package": {
+    "name": "filestream",
+    "version": ""
+  },
+  "name": "agent-monitor",
+  "namespace": "",
+  "description": "",
+  "policy_ids": [
+    "so-grid-nodes_general"
+  ],
+  "output_id": null,
+  "vars": {},
+  "inputs": {
+    "filestream-filestream": {
+      "enabled": true,
+      "streams": {
+        "filestream.generic": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/agents/agent-monitor.log"
+            ],
+            "data_stream.dataset": "agentmonitor",
+            "pipeline": "elasticagent.monitor",
+            "parsers": "",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- add_fields:\n    target: event\n    fields:\n        module: gridmetrics",
+            "tags": [],
+            "recursive_glob": true,
+            "ignore_older": "72h",
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": true,
+            "fingerprint_offset": 0,
+            "fingerprint_length": 64,
+            "file_identity_native": false,
+            "exclude_lines": [],
+            "include_lines": []
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/files/integrations/grid-nodes_general/elasticsearch-logs.json

@@ -40,7 +40,7 @@
           "enabled": true,
           "vars": {
             "paths": [
-              "/opt/so/log/elasticsearch/*.log"
+              "/opt/so/log/elasticsearch/*.json"
             ]
           }
         },

salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json

@@ -1,26 +1,43 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "hydra-logs",
-  "namespace": "so",
   "description": "Hydra logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/hydra/hydra.log"
             ],
             "data_stream.dataset": "hydra",
-            "tags": ["so-hydra"],
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: hydra",
-            "custom": "pipeline: hydra"
+            "pipeline": "hydra",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: hydra",
+            "tags": [
+              "so-hydra"
+            ],
+            "recursive_glob": true,
+            "ignore_older": "72h",
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
@@ -28,3 +45,5 @@
   },
   "force": true
 }
+
+

salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json

@@ -1,30 +1,44 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "idh-logs",
-  "namespace": "so",
   "description": "IDH integration",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/idh/opencanary.log"
             ],
             "data_stream.dataset": "idh",
+            "pipeline": "common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
+            "processors": "\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- convert:\n    fields:\n      - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n- drop_fields:\n    when:\n      equals:\n        event.code: \"1001\"\n    fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n    ignore_missing: true\n- rename:\n    fields:\n      - from: \"src_host\"\n        to: \"source.ip\"\n      - from: \"src_port\"\n        to: \"source.port\"\n      - from: \"dst_host\"\n        to: \"destination.host\"\n      - from: \"dst_port\"\n        to: \"destination.port\"\n    ignore_missing: true\n- drop_fields:\n    fields: '[\"prospector\", \"input\", \"offset\", \"beat\"]'\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: opencanary",
             "tags": [],
-            "processors": "\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- drop_fields:\n    when:\n      equals:\n        logtype: \"1001\"\n    fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n    ignore_missing: true\n- rename:\n    fields:\n      - from: \"src_host\"\n        to: \"source.ip\"\n      - from: \"src_port\"\n        to: \"source.port\"\n      - from: \"dst_host\"\n        to: \"destination.host\"\n      - from: \"dst_port\"\n        to: \"destination.port\"\n    ignore_missing: true\n- convert:\n    fields:\n      - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n    ignore_missing: true\n- drop_fields:\n    fields: '[\"prospector\", \"input\", \"offset\", \"beat\"]'\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: opencanary",
-            "custom": "pipeline: common"
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -1,33 +1,46 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "import-evtx-logs",
-  "namespace": "so",
   "description": "Import Windows EVTX logs",
   "policy_id": "so-grid-nodes_general",
-  "vars": {},
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/evtx/*.json"
             ],
             "data_stream.dataset": "import",
-            "custom": "",
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-1.67.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-2.5.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-1.67.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-1.67.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-2.5.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.6.1\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.1.2\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.6.1\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.6.1\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.1.2\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
-            ]
+            ],
+            "recursive_glob": true,
+            "ignore_older": "72h",
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/import-suricata-logs.json

@@ -1,30 +1,45 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "import-suricata-logs",
-  "namespace": "so",
   "description": "Import Suricata logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/import/*/suricata/eve*.json"
             ],
             "data_stream.dataset": "import",
+            "pipeline": "suricata.common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
+            "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata\n      imported: true\n- dissect:\n      tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n      field: \"log.file.path\"\n      target_prefix: \"\"\n",
             "tags": [],
-            "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata\n      imported: true\n- dissect:\n      tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n      field: \"log.file.path\"\n      target_prefix: \"\"",
-            "custom": "pipeline: suricata.common"
+            "recursive_glob": true,
+            "ignore_older": "72h",
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/kratos-logs.json

@@ -1,30 +0,0 @@
-{
-  "package": {
-    "name": "log",
-    "version": ""
-  },
-  "name": "kratos-logs",
-  "namespace": "so",
-  "description": "Kratos logs",
-  "policy_id": "so-grid-nodes_general",
-  "inputs": {
-    "logs-logfile": {
-      "enabled": true,
-      "streams": {
-        "log.logs": {
-          "enabled": true,
-          "vars": {
-            "paths": [
-              "/opt/so/log/kratos/kratos.log"
-            ],
-            "data_stream.dataset": "kratos",
-            "tags": ["so-kratos"],
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos",
-            "custom": "pipeline: kratos"
-          }
-        }
-      }
-    }
-  },
-  "force": true
-}

salt/elasticfleet/files/integrations/grid-nodes_general/redis-logs.json

@@ -15,7 +15,7 @@
           "enabled": true,
           "vars": {
             "paths": [
-              "/opt/so/log/redis/redis.log"
+              "/opt/so/log/redis/redis-server.log"
             ],
             "tags": [
               "redis-log"

salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json

@@ -1,18 +1,17 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "rita-logs",
-  "namespace": "so",
   "description": "RITA Logs",
   "policy_id": "so-grid-nodes_general",
-  "vars": {},
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
@@ -20,15 +19,28 @@
               "/nsm/rita/exploded-dns.csv",
               "/nsm/rita/long-connections.csv"
             ],
-            "exclude_files": [],
-            "ignore_older": "72h",
             "data_stream.dataset": "rita",
-            "tags": [],
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- dissect:\n    tokenizer: \"/nsm/rita/%{pipeline}.csv\"\n    field: \"log.file.path\"\n    trim_chars: \".csv\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\").split(\"-\");\n          if (pl.length > 1) {\n            pl = pl[1];\n          }\n          else {\n            pl = pl[0];\n          }\n          event.Put(\"@metadata.pipeline\", \"rita.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: rita",
-            "custom": "exclude_lines: ['^Score', '^Source', '^Domain', '^No results']"
+            "tags": [],
+            "recursive_glob": true,
+            "ignore_older": "72h",
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
-  }
+  },
+  "force": true
 }

salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json

@@ -1,29 +1,41 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "so-ip-mappings",
-  "namespace": "so",
   "description": "IP Description mappings",
   "policy_id": "so-grid-nodes_general",
-  "vars": {},
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/custom-mappings/ip-descriptions.csv"
             ],
             "data_stream.dataset": "hostnamemappings",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
+            "processors": "- decode_csv_fields:\n    fields:\n      message: decoded.csv\n    separator: \",\"\n    ignore_missing: false\n    overwrite_keys: true\n    trim_leading_space: true\n    fail_on_error: true\n\n- extract_array:\n    field: decoded.csv\n    mappings:\n      so.ip_address: '0'\n      so.description: '1'\n\n- script:\n    lang: javascript\n    source: >\n      function process(event) {\n          var ip = event.Get('so.ip_address');\n          var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n          if (!validIpRegex.test(ip)) {\n              event.Cancel();\n          }\n      }\n- fingerprint:\n    fields: [\"so.ip_address\"]\n    target_field: \"@metadata._id\"\n",
             "tags": [
               "so-ip-mappings"
             ],
-            "processors": "- decode_csv_fields:\n    fields:\n      message: decoded.csv\n    separator: \",\"\n    ignore_missing: false\n    overwrite_keys: true\n    trim_leading_space: true\n    fail_on_error: true\n\n- extract_array:\n    field: decoded.csv\n    mappings:\n      so.ip_address: '0'\n      so.description: '1'\n\n- script:\n    lang: javascript\n    source: >\n      function process(event) {\n          var ip = event.Get('so.ip_address');\n          var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n          if (!validIpRegex.test(ip)) {\n              event.Cancel();\n          }\n      }\n- fingerprint:\n    fields: [\"so.ip_address\"]\n    target_field: \"@metadata._id\"\n",
-            "custom": ""
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
@@ -31,5 +43,3 @@
   },
   "force": true
 }
-
-

salt/elasticfleet/files/integrations/grid-nodes_general/soc-auth-sync-logs.json

@@ -1,30 +1,44 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "soc-auth-sync-logs",
-  "namespace": "so",
   "description": "Security Onion - Elastic Auth Sync - Logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sync.log"
             ],
             "data_stream.dataset": "soc",
-            "tags": ["so-soc"],
+            "pipeline": "common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- dissect:\n    tokenizer: \"%{event.action}\"\n    field: \"message\"\n    target_prefix: \"\"\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: auth_sync",
-            "custom": "pipeline: common"
+            "tags": [],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json

@@ -1,35 +1,48 @@
 {
-  "policy_id": "so-grid-nodes_general",
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "soc-detections-logs",
   "description": "Security Onion Console - Detections Logs",
+  "policy_id": "so-grid-nodes_general",
   "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/detections_runtime-status_sigma.log",
               "/opt/so/log/soc/detections_runtime-status_yara.log"
             ],
-            "exclude_files": [],
-            "ignore_older": "72h",
             "data_stream.dataset": "soc",
+            "pipeline": "common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: detections\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true",
             "tags": [
               "so-soc"
             ],
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: detections\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true",
-            "custom": "pipeline: common"
+            "recursive_glob": true,
+            "ignore_older": "72h",
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/soc-salt-relay-logs.json

@@ -1,30 +1,46 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "soc-salt-relay-logs",
-  "namespace": "so",
   "description": "Security Onion - Salt Relay - Logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/salt-relay.log"
             ],
             "data_stream.dataset": "soc",
-            "tags": ["so-soc"],
+            "pipeline": "common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- dissect:\n    tokenizer: \"%{soc.ts} | %{event.action}\"\n    field: \"message\"\n    target_prefix: \"\"\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: salt_relay",
-            "custom": "pipeline: common"
+            "tags": [
+              "so-soc"
+            ],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/soc-sensoroni-logs.json

@@ -1,30 +1,44 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "soc-sensoroni-logs",
-  "namespace": "so",
   "description": "Security Onion - Sensoroni - Logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/sensoroni/sensoroni.log"
             ],
             "data_stream.dataset": "soc",
-            "tags": [],
+            "pipeline": "common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"sensoroni\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: sensoroni\n- rename:\n    fields:\n      - from: \"sensoroni.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"sensoroni.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"sensoroni.fields.method\"\n        to: \"http.request.method\"\n      - from: \"sensoroni.fields.path\"\n        to: \"url.path\"\n      - from: \"sensoroni.message\"\n        to: \"event.action\"\n      - from: \"sensoroni.level\"\n        to: \"log.level\"\n    ignore_missing: true",
-            "custom": "pipeline: common"
+            "tags": [],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
-  "force": true
-}
+"force": true
+}

salt/elasticfleet/files/integrations/grid-nodes_general/soc-server-logs.json

@@ -1,30 +1,46 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "soc-server-logs",
-  "namespace": "so",
   "description": "Security Onion Console Logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/opt/so/log/soc/sensoroni-server.log"
             ],
             "data_stream.dataset": "soc",
-            "tags": ["so-soc"],
+            "pipeline": "common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: server\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true",
-            "custom": "pipeline: common"
+            "tags": [
+              "so-soc"
+            ],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/strelka-logs.json

@@ -1,30 +1,44 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "strelka-logs",
-  "namespace": "so",
-  "description": "Strelka logs",
+  "description": "Strelka Logs",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/strelka/log/strelka.log"
             ],
             "data_stream.dataset": "strelka",
-            "tags": [],
+            "pipeline": "strelka.file",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- add_fields:\n    target: event\n    fields:\n      category: file\n      module: strelka",
-            "custom": "pipeline: strelka.file"
+            "tags": [],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }
     }
   },
   "force": true
-}
+}

salt/elasticfleet/files/integrations/grid-nodes_general/suricata-logs.json

@@ -1,26 +1,40 @@
 {
   "package": {
-    "name": "log",
+    "name": "filestream",
     "version": ""
   },
   "name": "suricata-logs",
-  "namespace": "so",
   "description": "Suricata integration",
   "policy_id": "so-grid-nodes_general",
+  "namespace": "so",
   "inputs": {
-    "logs-logfile": {
+    "filestream-filestream": {
       "enabled": true,
       "streams": {
-        "log.logs": {
+        "filestream.generic": {
           "enabled": true,
           "vars": {
             "paths": [
               "/nsm/suricata/eve*.json"
             ],
             "data_stream.dataset": "suricata",
-            "tags": [],
+            "pipeline": "suricata.common",
+            "parsers": "#- ndjson:\n#    target: \"\"\n#    message_key: msg\n#- multiline:\n#    type: count\n#    count_lines: 3\n",
+            "exclude_files": [
+              "\\.gz$"
+            ],
+            "include_files": [],
             "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata",
-            "custom": "pipeline: suricata.common"
+            "tags": [],
+            "recursive_glob": true,
+            "clean_inactive": -1,
+            "harvester_limit": 0,
+            "fingerprint": false,
+            "fingerprint_offset": 0,
+            "fingerprint_length": "64",
+            "file_identity_native": true,
+            "exclude_lines": [],
+            "include_lines": []
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json

@@ -11,7 +11,7 @@
     "tcp-tcp": {
       "enabled": true,
       "streams": {
-        "tcp.generic": {
+        "tcp.tcp": {
           "enabled": true,
           "vars": {
             "listen_address": "0.0.0.0",
@@ -23,7 +23,8 @@
               "syslog"
             ],
             "syslog_options": "field: message\n#format: auto\n#timezone: Local",
-            "ssl": ""
+            "ssl": "",
+            "custom": ""
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json

@@ -31,7 +31,8 @@
             ],
             "tags": [
               "so-grid-node"
-            ]
+            ],
+            "processors": "- if:\n    contains:\n      message: \"salt-minion\"\n  then: \n    - dissect:\n        tokenizer: \"%{} %{} %{} %{} %{} %{}: [%{log.level}] %{*}\"\n        field: \"message\"\n        trim_values: \"all\"\n        target_prefix: \"\"\n    - drop_event:\n        when:\n          equals:\n            log.level: \"INFO\""
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_heavy/elasticsearch-grid-nodes.json

@@ -0,0 +1,107 @@
+{
+  "package": {
+    "name": "elasticsearch",
+    "version": ""
+  },
+  "name": "elasticsearch-grid-nodes_heavy",
+  "namespace": "default",
+  "description": "Elasticsearch Logs",
+  "policy_id": "so-grid-nodes_heavy",
+  "inputs": {
+    "elasticsearch-logfile": {
+      "enabled": true,
+      "streams": {
+        "elasticsearch.audit": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/*_audit.json"
+            ]
+          }
+        },
+        "elasticsearch.deprecation": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/*_deprecation.json"
+            ]
+          }
+        },
+        "elasticsearch.gc": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/gc.log.[0-9]*",
+              "/var/log/elasticsearch/gc.log"
+            ]
+          }
+        },
+        "elasticsearch.server": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/elasticsearch/*.json"
+            ]
+          }
+        },
+        "elasticsearch.slowlog": {
+          "enabled": false,
+          "vars": {
+            "paths": [
+              "/var/log/elasticsearch/*_index_search_slowlog.json",
+              "/var/log/elasticsearch/*_index_indexing_slowlog.json"
+            ]
+          }
+        }
+      }
+    },
+    "elasticsearch-elasticsearch/metrics": {
+      "enabled": false,
+      "vars": {
+        "hosts": [
+          "http://localhost:9200"
+        ],
+        "scope": "node"
+      },
+      "streams": {
+        "elasticsearch.stack_monitoring.ccr": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.cluster_stats": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.enrich": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.index": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.index_recovery": {
+          "enabled": false,
+          "vars": {
+            "active.only": true
+          }
+        },
+        "elasticsearch.stack_monitoring.index_summary": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.ml_job": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.node": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.node_stats": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.pending_tasks": {
+          "enabled": false
+        },
+        "elasticsearch.stack_monitoring.shard": {
+          "enabled": false
+        }
+      }
+    }
+  },
+  "force": true
+}

salt/elasticfleet/install_agent_grid.sls

@@ -2,26 +2,32 @@
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.
 
-{%- set GRIDNODETOKENGENERAL = salt['pillar.get']('global:fleet_grid_enrollment_token_general') -%}
-{%- set GRIDNODETOKENHEAVY = salt['pillar.get']('global:fleet_grid_enrollment_token_heavy') -%}
+{% set GRIDNODETOKEN = salt['pillar.get']('global:fleet_grid_enrollment_token_general') -%}
+{% if grains.role == 'so-heavynode' %}
+{%   set GRIDNODETOKEN = salt['pillar.get']('global:fleet_grid_enrollment_token_heavy') -%}
+{% endif %}
 
 {% set AGENT_STATUS = salt['service.available']('elastic-agent') %}
-{% if not AGENT_STATUS  %}
+{% set AGENT_EXISTS = salt['file.file_exists']('/opt/Elastic/Agent/elastic-agent') %}
 
-{% if grains.role not in ['so-heavynode'] %}
-run_installer:
-  cmd.script:
-    - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
-    - cwd: /opt/so
-    - args: -token={{ GRIDNODETOKENGENERAL }}
-    - retry: True
-{% else %} 
-run_installer:
-  cmd.script:
-    - name: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
-    - cwd: /opt/so
-    - args: -token={{ GRIDNODETOKENHEAVY }}
-    - retry: True
-{% endif %}  
+{% if not AGENT_STATUS or not AGENT_EXISTS %}
 
+pull_agent_installer:
+  file.managed:
+    - name: /opt/so/so-elastic-agent_linux_amd64
+    - source: salt://elasticfleet/files/so_agent-installers/so-elastic-agent_linux_amd64
+    - mode: 755
+    - makedirs: True
+
+run_installer:
+  cmd.run:
+    - name: ./so-elastic-agent_linux_amd64 -token={{ GRIDNODETOKEN }} -force
+    - cwd: /opt/so
+    - retry:
+        attempts: 3
+        interval: 20
+
+cleanup_agent_installer:
+  file.absent:
+    - name: /opt/so/so-elastic-agent_linux_amd64
 {% endif %}

salt/elasticfleet/integration-defaults.map.jinja

@@ -4,6 +4,7 @@
 
 
 {% import_json '/opt/so/state/esfleet_package_components.json' as ADDON_PACKAGE_COMPONENTS %}
+{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
 {% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
 
 {% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
@@ -14,11 +15,13 @@
     'awsfirehose.logs': 'awsfirehose',
     'awsfirehose.metrics': 'aws.cloudwatch',
     'cribl.logs': 'cribl',
+    'cribl.metrics': 'cribl',
     'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
     'azure_application_insights.app_insights': 'azure.app_insights',
     'azure_application_insights.app_state': 'azure.app_state',
     'azure_billing.billing': 'azure.billing',
     'azure_functions.metrics': 'azure.function',
+    'azure_ai_foundry.metrics': 'azure.ai_foundry',
     'azure_metrics.compute_vm_scaleset': 'azure.compute_vm_scaleset',
     'azure_metrics.compute_vm': 'azure.compute_vm',
     'azure_metrics.container_instance': 'azure.container_instance',
@@ -45,7 +48,10 @@
     'synthetics.browser_screenshot': 'synthetics-browser.screenshot',
     'synthetics.http': 'synthetics-http',
     'synthetics.icmp': 'synthetics-icmp',
-    'synthetics.tcp': 'synthetics-tcp'
+    'synthetics.tcp': 'synthetics-tcp',
+    'swimlane.swimlane_api': 'swimlane.api',
+    'swimlane.tenant_api': 'swimlane.tenant',
+    'swimlane.turbine_api': 'turbine.api'
    } %}
 
 {% for pkg in ADDON_PACKAGE_COMPONENTS %}
@@ -62,70 +68,112 @@
 {%         else %}
 {%           set integration_type = "" %}
 {%         endif %}
-{%           set component_name = pkg.name ~ "." ~ pattern.title %}
-{#           fix weirdly named components #}
-{%           if component_name in WEIRD_INTEGRATIONS %}
-{%             set component_name = WEIRD_INTEGRATIONS[component_name] %}
-{%           endif %}
+{%         set component_name = pkg.name ~ "." ~ pattern.title %}
+{%         set index_pattern = pattern.name %}
+
+{#         fix weirdly named components #}
+{%         if component_name in WEIRD_INTEGRATIONS %}
+{%           set component_name = WEIRD_INTEGRATIONS[component_name] %}
+{%         endif %}
+
+{#         create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
+           eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
+{%         set custom_component_name = component_name %}
+
+{#         duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
+{%         set generic_integration_type = integration_type %}
+
 {#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
 {%           set component_name_x = component_name.replace(".","_x_") %}
 {#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
 {%           set integration_key = "so-" ~ integration_type ~ component_name_x %}
 
+{#         if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #}
+{%         if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %}
+{#           these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
+{%           set index_pattern = integration_type ~ component_name ~ "-*" %}
+{#           includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
+{%           set component_name = "filestream.generic" %}
+{%           set generic_integration_type = "logs-" %}
+{%         endif %}
+
 {#           Default integration settings #}
 {%           set integration_defaults = {
-              "index_sorting": false,
-              "index_template": {
-                  "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
-                  "data_stream": {
-                      "allow_custom_routing": false,
-                      "hidden": false
-                  },
-                  "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
-                  "index_patterns": [pattern.name],
-                  "priority": 501,
-                  "template": {
-                      "settings": {
-                          "index": {
-                              "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
-                              "number_of_replicas": 0
-                          }
-                      }
-                  }
-              },
-              "policy": {
-                  "phases": {
-                      "cold": {
-                          "actions": {
-                              "set_priority": {"priority": 0}
-                          },
-                          "min_age": "60d"
+               "index_sorting": false,
+               "index_template": {
+                 "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "data_stream": {
+                   "allow_custom_routing": false,
+                   "hidden": false
+                 },
+                 "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
+                 "index_patterns": [index_pattern],
+                 "priority": 501,
+                 "template": {
+                   "settings": {
+                     "index": {
+                       "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
+                         "number_of_replicas": 0
+                     }
+                   }
+                 }
+               },
+               "policy": {
+                 "phases": {
+                   "cold": {
+                     "actions": {
+                        "allocate":{
+                          "number_of_replicas": ""
+                        },
+                       "set_priority": {"priority": 0}
+                     },
+                     "min_age": "60d"
+                   },
+                   "delete": {
+                     "actions": {
+                       "delete": {}
+                     },
+                     "min_age": "365d"
+                   },
+                   "hot": {
+                     "actions": {
+                       "rollover": {
+                         "max_age": "30d",
+                         "max_primary_shard_size": "50gb"
+                       },
+                       "forcemerge":{
+                         "max_num_segments": ""
+                       },
+                       "shrink":{
+                         "max_primary_shard_size": "",
+                         "method": "COUNT",
+                         "number_of_shards": ""
+                       },
+                       "set_priority": {"priority": 100}
                       },
-                      "delete": {
-                          "actions": {
-                              "delete": {}
-                          },
-                          "min_age": "365d"
-                      },
-                      "hot": {
-                          "actions": {
-                              "rollover": {
-                                  "max_age": "30d",
-                                  "max_primary_shard_size": "50gb"
-                              },
-                              "set_priority": {"priority": 100}
-                          },
-                          "min_age": "0ms"
-                      },
-                      "warm": {
-                          "actions": {
-                              "set_priority": {"priority": 50}
-                          },
-                          "min_age": "30d"
-                      }
-                  }
-              }
-          } %}
+                      "min_age": "0ms"
+                   },
+                   "warm": {
+                     "actions": {
+                        "allocate": {
+                          "number_of_replicas": ""
+                        },
+                        "forcemerge": {
+                          "max_num_segments": ""
+                        },
+                        "shrink":{
+                          "max_primary_shard_size": "",
+                          "method": "COUNT",
+                          "number_of_shards": ""
+                        },
+                       "set_priority": {"priority": 50}
+                     },
+                     "min_age": "30d"
+                   }
+                 }
+               }
+             } %}
+
 {%         do ADDON_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
 {%       endfor %}
 {%     endif %}

salt/elasticfleet/soc_elasticfleet.yaml

@@ -45,6 +45,51 @@ elasticfleet:
       global: True
       forcedType: bool
       helpLink: elastic-fleet.html
+    auto_upgrade_integrations:
+      description: Enables or disables automatically upgrading Elastic Agent integrations.
+      global: True
+      forcedType: bool
+      helpLink: elastic-fleet.html
+    outputs:
+      logstash:
+        bulk_max_size:
+          description: The maximum number of events to bulk in a single Logstash request.
+          global: True
+          forcedType: int
+          advanced: True
+          helpLink: elastic-fleet.html
+        worker:
+          description: The number of workers per configured host publishing events.
+          global: True
+          forcedType: int
+          advanced: true
+          helpLink: elastic-fleet.html
+        queue_mem_events:
+          title: queued events
+          description: The number of events the queue can store. This value should be evenly divisible by the smaller of 'bulk_max_size' to avoid sending partial batches to the output.
+          global: True
+          forcedType: int
+          advanced: True
+          helpLink: elastic-fleet.html
+        timeout:
+          description: The number of seconds to wait for responses from the Logstash server before timing out. Eg 30s
+          regex: ^[0-9]+s$
+          advanced: True
+          global: True
+          helpLink: elastic-fleet.html
+        loadbalance:
+          description: If true and multiple Logstash hosts are configured, the output plugin load balances published events onto all Logstash hosts. If false, the output plugin sends all events to one host (determined at random) and switches to another host if the selected one becomes unresponsive.
+          forcedType: bool
+          advanced: True
+          global: True
+          helpLink: elastic-fleet.html
+        compression_level:
+          description: The gzip compression level. The compression level must be in the range of 1 (best speed) to 9 (best compression).
+          regex: ^[1-9]$
+          forcedType: int
+          advanced: True
+          global: True
+          helpLink: elastic-fleet.html
     server:
       custom_fqdn:
         description: Custom FQDN for Agents to connect to. One per line.

salt/elasticfleet/ssl.sls

@@ -0,0 +1,186 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%   from 'ca/map.jinja' import CA %}
+
+{% if GLOBALS.is_manager or GLOBALS.role in ['so-heavynode', 'so-fleet', 'so-receiver'] %}
+
+{% if grains['role'] not in [ 'so-heavynode', 'so-receiver'] %}
+# Start -- Elastic Fleet Host Cert
+etc_elasticfleet_key:
+  x509.private_key_managed:
+    - name: /etc/pki/elasticfleet-server.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/elasticfleet-server.key') -%}
+    - prereq:
+      - x509: etc_elasticfleet_crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+etc_elasticfleet_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/elasticfleet-server.crt
+    - ca_server: {{ CA.server }}
+    - signing_policy: elasticfleet
+    - private_key: /etc/pki/elasticfleet-server.key
+    - CN: {{ GLOBALS.hostname }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }},DNS:{{ GLOBALS.url_base }},IP:{{ GLOBALS.node_ip }}{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %},DNS:{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(',DNS:') }}{% endif %}
+    - days_remaining: 7
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+
+efperms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-server.key
+    - mode: 640
+    - group: 939
+
+chownelasticfleetcrt:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-server.crt
+    - mode: 640
+    - user: 947
+    - group: 939
+
+chownelasticfleetkey:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-server.key
+    - mode: 640
+    - user: 947
+    - group: 939
+# End -- Elastic Fleet Host Cert
+{% endif %} # endif is for not including HeavyNodes & Receivers 
+
+
+# Start -- Elastic Fleet Client Cert for Agent (Mutual Auth with Logstash Output)
+etc_elasticfleet_agent_key:
+  x509.private_key_managed:
+    - name: /etc/pki/elasticfleet-agent.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/elasticfleet-agent.key') -%}
+    - prereq:
+      - x509: etc_elasticfleet_agent_crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+etc_elasticfleet_agent_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/elasticfleet-agent.crt
+    - ca_server: {{ CA.server }}
+    - signing_policy: elasticfleet
+    - private_key: /etc/pki/elasticfleet-agent.key
+    - CN: {{ GLOBALS.hostname }}
+    - days_remaining: 7
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+  cmd.run:
+    - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-agent.key -topk8 -out /etc/pki/elasticfleet-agent.p8 -nocrypt"
+    - onchanges:
+      - x509: etc_elasticfleet_agent_key
+
+efagentperms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-agent.key
+    - mode: 640
+    - group: 939
+
+chownelasticfleetagentcrt:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-agent.crt
+    - mode: 640
+    - user: 947
+    - group: 939
+
+chownelasticfleetagentkey:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-agent.key
+    - mode: 640
+    - user: 947
+    - group: 939
+# End -- Elastic Fleet Client Cert for Agent (Mutual Auth with Logstash Output)
+
+{% endif %}
+
+{% if GLOBALS.role in ['so-manager', 'so-managerhype', 'so-managersearch', 'so-standalone'] %}
+elasticfleet_kafka_key:
+  x509.private_key_managed:
+    - name: /etc/pki/elasticfleet-kafka.key
+    - keysize: 4096
+    - backup: True
+    - new: True
+    {% if salt['file.file_exists']('/etc/pki/elasticfleet-kafka.key') -%}
+    - prereq:
+      - x509: elasticfleet_kafka_crt
+    {%- endif %}
+    - retry:
+        attempts: 5
+        interval: 30
+
+elasticfleet_kafka_crt:
+  x509.certificate_managed:
+    - name: /etc/pki/elasticfleet-kafka.crt
+    - ca_server: {{ CA.server }}
+    - signing_policy: kafka
+    - private_key: /etc/pki/elasticfleet-kafka.key
+    - CN: {{ GLOBALS.hostname }}
+    - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }}
+    - days_remaining: 7
+    - days_valid: 820
+    - backup: True
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 30
+
+elasticfleet_kafka_cert_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-kafka.crt
+    - mode: 640
+    - user: 947
+    - group: 939
+
+elasticfleet_kafka_key_perms:
+  file.managed:
+    - replace: False
+    - name: /etc/pki/elasticfleet-kafka.key
+    - mode: 640
+    - user: 947
+    - group: 939
+{% endif %}
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -23,6 +23,13 @@ fi
 # Define a banner to separate sections
 banner="========================================================================="
 
+fleet_api() {
+    local QUERYPATH=$1
+    shift
+
+    curl -sK /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/${QUERYPATH}" "$@" --retry 3 --retry-delay 10 --fail 2>/dev/null
+}
+
 elastic_fleet_integration_check() {
 
     AGENT_POLICY=$1
@@ -39,7 +46,9 @@ elastic_fleet_integration_create() {
 
     JSON_STRING=$1
 
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPOST -d "$JSON_STRING"; then
+        return 1
+    fi
 }
 
 
@@ -56,7 +65,10 @@ elastic_fleet_integration_remove() {
                     '{"packagePolicyIds":[$INTEGRATIONID]}'
                     )
 
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies/delete" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "package_policies/delete" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+        echo "Error: Unable to delete '$NAME' from '$AGENT_POLICY'"
+        return 1
+    fi
 }
 
 elastic_fleet_integration_update() {
@@ -65,7 +77,9 @@ elastic_fleet_integration_update() {
 
     JSON_STRING=$2
 
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "package_policies/$UPDATE_ID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -XPUT -d "$JSON_STRING"; then
+        return 1
+    fi
 }
 
 elastic_fleet_integration_policy_upgrade() {
@@ -77,101 +91,116 @@ elastic_fleet_integration_policy_upgrade() {
                     '{"packagePolicyIds":[$INTEGRATIONID]}'
                     )
 
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies/upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "package_policies/upgrade" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+        return 1
+    fi
 }
 
 
 elastic_fleet_package_version_check() {
     PACKAGE=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.version'
+
+    if output=$(fleet_api "epm/packages/$PACKAGE"); then
+        echo "$output" | jq -r '.item.version'
+    else
+        echo "Error: Failed to get current package version for '$PACKAGE'"
+        return 1
+    fi
 }
 
 elastic_fleet_package_latest_version_check() {
     PACKAGE=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.latestVersion'
+    if output=$(fleet_api "epm/packages/$PACKAGE"); then
+        if version=$(jq -e -r '.item.latestVersion' <<< $output); then
+            echo "$version"
+        fi
+    else
+        echo "Error: Failed to get latest version for '$PACKAGE'"
+        return 1
+    fi
 }
 
 elastic_fleet_package_install() {
     PKG=$1
     VERSION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  -d '{"force":true}' "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION"
+    if ! fleet_api "epm/packages/$PKG/$VERSION" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"force":true}'; then
+        return 1
+    fi
 }
 
 elastic_fleet_bulk_package_install() {
     BULK_PKG_LIST=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$1 "localhost:5601/api/fleet/epm/packages/_bulk"
-}
-
-elastic_fleet_package_is_installed() {
-    PACKAGE=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.status'
-}
-
-elastic_fleet_installed_packages() {
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=500"
-}
-
-elastic_fleet_agent_policy_ids() {
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].id
-    if [ $? -ne 0 ]; then
-        echo "Error: Failed to retrieve agent policies."
-        exit 1
+    if ! fleet_api "epm/packages/_bulk" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$BULK_PKG_LIST; then
+        return 1
     fi
 }
 
-elastic_fleet_agent_policy_names() {
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].name
-    if [ $? -ne 0 ]; then
+elastic_fleet_installed_packages() {
+    if ! fleet_api "epm/packages/installed?perPage=500"; then
+        return 1
+    fi
+}
+
+elastic_fleet_agent_policy_ids() {
+    if output=$(fleet_api "agent_policies"); then
+        echo "$output" | jq -r .items[].id
+    else
         echo "Error: Failed to retrieve agent policies."
-        exit 1
+        return 1
     fi
 }
 
 elastic_fleet_integration_policy_names() {
     AGENT_POLICY=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r .item.package_policies[].name
-    if [ $? -ne 0 ]; then
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
+        echo "$output" | jq -r .item.package_policies[].name
+    else
         echo "Error: Failed to retrieve integrations for '$AGENT_POLICY'."
-        exit 1
+        return 1
     fi
 }
 
 elastic_fleet_integration_policy_package_name() {
     AGENT_POLICY=$1
     INTEGRATION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.name'
-    if [ $? -ne 0 ]; then
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
+        echo "$output" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.name'
+    else
         echo "Error: Failed to retrieve package name for '$INTEGRATION' in '$AGENT_POLICY'."
-        exit 1
+        return 1
     fi
 }
 
 elastic_fleet_integration_policy_package_version() {
     AGENT_POLICY=$1
     INTEGRATION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version'
-    if [ $? -ne 0 ]; then
-        echo "Error: Failed to retrieve package version for '$INTEGRATION' in '$AGENT_POLICY'."
-        exit 1
+
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
+        if version=$(jq -e -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version' <<< "$output"); then
+            echo "$version"
+        fi
+    else
+        echo "Error: Failed to retrieve integration version for '$INTEGRATION' in policy '$AGENT_POLICY'"
+        return 1
     fi
 }
 
 elastic_fleet_integration_id() {
     AGENT_POLICY=$1
     INTEGRATION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .id'
-    if [ $? -ne 0 ]; then
+    if output=$(fleet_api "agent_policies/$AGENT_POLICY"); then
+        echo "$output" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .id'
+    else
         echo "Error: Failed to retrieve integration ID for '$INTEGRATION' in '$AGENT_POLICY'."
-        exit 1
+        return 1
     fi
 }
 
 elastic_fleet_integration_policy_dryrun_upgrade() {
     INTEGRATION_ID=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -H "Content-Type: application/json" -H 'kbn-xsrf: true' -L -X POST "localhost:5601/api/fleet/package_policies/upgrade/dryrun" -d "{\"packagePolicyIds\":[\"$INTEGRATION_ID\"]}"
-    if [ $? -ne 0 ]; then
+    if ! fleet_api "package_policies/upgrade/dryrun" -H "Content-Type: application/json" -H 'kbn-xsrf: true' -XPOST -d "{\"packagePolicyIds\":[\"$INTEGRATION_ID\"]}"; then
         echo "Error: Failed to complete dry run for '$INTEGRATION_ID'."
-        exit 1
+        return 1
     fi
 }
 
@@ -180,25 +209,18 @@ elastic_fleet_policy_create() {
     NAME=$1
     DESC=$2
     FLEETSERVER=$3
-        TIMEOUT=$4
+    TIMEOUT=$4
 
     JSON_STRING=$( jq -n \
-                    --arg NAME "$NAME" \
-                    --arg DESC "$DESC" \
-                    --arg TIMEOUT $TIMEOUT \
-                    --arg FLEETSERVER "$FLEETSERVER" \
-                    '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
-                    )
+        --arg NAME "$NAME" \
+        --arg DESC "$DESC" \
+        --arg TIMEOUT $TIMEOUT \
+        --arg FLEETSERVER "$FLEETSERVER" \
+            '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":$TIMEOUT,"has_fleet_server":$FLEETSERVER}'
+        )
     # Create Fleet Policy
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+    if ! fleet_api "agent_policies" -XPOST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"; then
+        return 1
+    fi
 
 }
-
-elastic_fleet_policy_update() {
-
-    POLICYID=$1
-    JSON_STRING=$2
-
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-}
-

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-defend

@@ -8,6 +8,7 @@
 
 . /usr/sbin/so-elastic-fleet-common
 
+ERROR=false
 # Manage Elastic Defend Integration for Initial Endpoints Policy
 for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/elastic-defend/*.json
 do
@@ -15,9 +16,20 @@ do
   elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
   if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Upgrading integration policy\n"
-      elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
+      if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then
+        echo -e "\nFailed to upgrade integration policy for ${INTEGRATION##*/}"
+        ERROR=true
+        continue
+      fi
   else
     printf "\n\nIntegration does not exist - Creating integration\n"
-    elastic_fleet_integration_create "@$INTEGRATION"
+    if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        ERROR=true
+        continue
+    fi
   fi
 done
+if [[ "$ERROR" == "true" ]]; then
+    exit 1
+fi

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server

@@ -25,5 +25,9 @@ for POLICYNAME in $POLICY; do
     .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
 
     # Now update the integration policy using the modified JSON
-    elastic_fleet_integration_update "$INTEGRATION_ID" "$UPDATED_INTEGRATION_POLICY"
+    if ! elastic_fleet_integration_update "$INTEGRATION_ID" "$UPDATED_INTEGRATION_POLICY"; then
+        # exit 1 on failure to update fleet integration policies, let salt handle retries
+        echo "Failed to update $POLICYNAME.."
+        exit 1
+    fi
 done

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -13,79 +13,101 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
   /usr/sbin/so-elastic-fleet-package-upgrade
 
   # Second, update Fleet Server policies
-  /sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
+  /usr/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
 
   # Third, configure Elastic Defend Integration seperately
   /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend
 
   # Initial Endpoints
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json
-  do
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/endpoints-initial/*.json; do
     printf "\n\nInitial Endpoints Policy - Loading $INTEGRATION\n"
     elastic_fleet_integration_check "endpoints-initial" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Updating integration\n"
-      elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
-      elastic_fleet_integration_create "@$INTEGRATION"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
     fi
   done
 
   # Grid Nodes - General
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json
-  do
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_general/*.json; do
     printf "\n\nGrid Nodes Policy_General - Loading $INTEGRATION\n"
     elastic_fleet_integration_check "so-grid-nodes_general" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Updating integration\n"
-      elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
-      elastic_fleet_integration_create "@$INTEGRATION"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
     fi
   done
-  if [[ "$RETURN_CODE" != "1" ]]; then
-    touch /opt/so/state/eaintegrations.txt
-  fi
 
   # Grid Nodes - Heavy
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json
-  do
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations/grid-nodes_heavy/*.json; do
     printf "\n\nGrid Nodes Policy_Heavy - Loading $INTEGRATION\n"
     elastic_fleet_integration_check "so-grid-nodes_heavy" "$INTEGRATION"
     if [ -n "$INTEGRATION_ID" ]; then
       printf "\n\nIntegration $NAME exists - Updating integration\n"
-      elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+      if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+        echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
+      fi
     else
       printf "\n\nIntegration does not exist - Creating integration\n"
-      if [ "$NAME" != "elasticsearch-logs" ]; then
-        elastic_fleet_integration_create "@$INTEGRATION"
+      if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+        echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+        RETURN_CODE=1
+        continue
       fi
     fi
   done
-  if [[ "$RETURN_CODE" != "1" ]]; then
-    touch /opt/so/state/eaintegrations.txt
-  fi
 
   # Fleet Server - Optional integrations
-  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json
-  do
+  for INTEGRATION in /opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json; do
     if ! [ "$INTEGRATION" == "/opt/so/conf/elastic-fleet/integrations-optional/FleetServer*/*.json" ]; then
       FLEET_POLICY=`echo "$INTEGRATION"| cut -d'/' -f7`
       printf "\n\nFleet Server Policy - Loading $INTEGRATION\n"
       elastic_fleet_integration_check "$FLEET_POLICY" "$INTEGRATION"
       if [ -n "$INTEGRATION_ID" ]; then
         printf "\n\nIntegration $NAME exists - Updating integration\n"
-        elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"
+        if ! elastic_fleet_integration_update "$INTEGRATION_ID" "@$INTEGRATION"; then
+          echo -e "\nFailed to update integration for ${INTEGRATION##*/}"
+          RETURN_CODE=1
+          continue
+        fi
       else
         printf "\n\nIntegration does not exist - Creating integration\n"
         if [ "$NAME" != "elasticsearch-logs" ]; then
-          elastic_fleet_integration_create "@$INTEGRATION"
+          if ! elastic_fleet_integration_create "@$INTEGRATION"; then
+            echo -e "\nFailed to create integration for ${INTEGRATION##*/}"
+            RETURN_CODE=1
+            continue
+          fi
         fi
       fi
     fi
   done
+
+  # Only create the state file if all policies were created/updated successfully
   if [[ "$RETURN_CODE" != "1" ]]; then
     touch /opt/so/state/eaintegrations.txt
   fi

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade

@@ -1,62 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-elastic-fleet-common
-
-curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/)
-if [ $? -ne 0 ]; then
-    echo "Error: Failed to connect to Kibana."
-    exit 1
-fi
-
-IFS=$'\n'
-agent_policies=$(elastic_fleet_agent_policy_ids)
-if [ $? -ne 0 ]; then
-    echo "Error: Failed to retrieve agent policies."
-    exit 1
-fi
-
-for AGENT_POLICY in $agent_policies; do
-    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
-    for INTEGRATION in $integrations; do
-        if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
-            # Get package name so we know what package to look for when checking the current and latest available version
-            PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
-
-            # Get currently installed version of package
-            PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION")
-
-            # Get latest available version of package
-            AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME")
-
-            # Get integration ID
-            INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION")
-
-            if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
-                # Dry run of the upgrade
-                echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
-                echo "Upgrading $INTEGRATION..."
-                echo "Starting dry run..."
-                DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID")
-                DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
-                
-                # If no errors with dry run, proceed with actual upgrade
-                if [[ "$DRYRUN_ERRORS" == "false" ]]; then
-                    echo "No errors detected. Proceeding with upgrade..."
-                    elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
-                    if [ $? -ne 0 ]; then
-                        echo "Error: Upgrade failed for integration ID '$INTEGRATION_ID'."
-                        exit 1
-                    fi
-                else
-                    echo "Errors detected during dry run. Stopping upgrade..."
-                    exit 1
-                fi
-            fi
-        fi
-    done
-done
-echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -83,5 +83,10 @@ docker run \
 {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
 printf "\n### MSI Generated...\n"
 
-printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n"
+printf "\n### Cleaning up temp files \n"
 rm -rf /nsm/elastic-agent-workspace
+rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
+
+printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n"
+\cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/
+chmod 644 /nsm/elastic-fleet/so_agent-installers/*

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade

@@ -14,7 +14,7 @@ if ! is_manager_node; then
 fi
 
 # Get current list of Grid Node Agents that need to be upgraded
-RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=policy_id%20%3A%20so-grid-nodes_%2A&showInactive=false&showUpgradeable=true&getStatusSummary=true")
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%3A%20{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%20AND%20policy_id%3A%20so-grid-nodes_%2A&showInactive=false&getStatusSummary=true" --retry 3 --retry-delay 30 --fail 2>/dev/null)
 
 # Check to make sure that the server responded with good data - else, bail from script
 CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON")

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update

@@ -26,7 +26,7 @@ function update_es_urls() {
 }
 
 # Get current list of Fleet Elasticsearch URLs
-RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_elasticsearch')
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_elasticsearch' --retry 3 --retry-delay 30 --fail 2>/dev/null)
 
 # Check to make sure that the server responded with good data - else, bail from script
 CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade

@@ -0,0 +1,95 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
+{%- set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %}
+
+. /usr/sbin/so-elastic-fleet-common
+
+curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/)
+if [ $? -ne 0 ]; then
+    echo "Error: Failed to connect to Kibana."
+    exit 1
+fi
+
+IFS=$'\n'
+agent_policies=$(elastic_fleet_agent_policy_ids)
+if [ $? -ne 0 ]; then
+    echo "Error: Failed to retrieve agent policies."
+    exit 1
+fi
+
+default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
+
+ERROR=false
+for AGENT_POLICY in $agent_policies; do
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then
+        # this script upgrades default integration packages, exit 1 and let salt handle retrying
+        exit 1
+    fi
+    for INTEGRATION in $integrations; do
+        if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
+            # Get package name so we know what package to look for when checking the current and latest available version
+            if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then
+                exit 1
+            fi
+            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+            if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
+            {%- endif %}
+                # Get currently installed version of package
+                attempt=0
+                max_attempts=3
+                while [ $attempt -lt $max_attempts ]; do
+                    if PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION") && AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME"); then
+                        break
+                    fi
+                    attempt=$((attempt + 1))
+                done
+                if [ $attempt -eq $max_attempts ]; then
+                    echo "Error: Failed getting $PACKAGE_VERSION or $AVAILABLE_VERSION"
+                    exit 1
+                fi
+
+                # Get integration ID
+                if ! INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION"); then
+                    exit 1
+                fi
+
+                if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
+                    # Dry run of the upgrade
+                    echo ""
+                    echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
+                    echo "Upgrading $INTEGRATION..."
+                    echo "Starting dry run..."
+                    if ! DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID"); then
+                        exit 1
+                    fi
+                    DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
+
+                    # If no errors with dry run, proceed with actual upgrade
+                    if [[ "$DRYRUN_ERRORS" == "false" ]]; then
+                        echo "No errors detected. Proceeding with upgrade..."
+                        if ! elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"; then
+                            echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'."
+                            ERROR=true
+                            continue
+                        fi
+                    else
+                        echo "Errors detected during dry run for $PACKAGE_NAME policy upgrade..."
+                        ERROR=true
+                        continue
+                    fi
+                fi
+            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+            fi
+            {%- endif %}
+        fi
+    done
+done
+if [[ "$ERROR" == "true" ]]; then
+    exit 1
+fi
+echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -3,7 +3,10 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
 {% set SUB = salt['pillar.get']('elasticfleet:config:subscription_integrations', default=false) %}
+{% set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %}
+{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
 
 . /usr/sbin/so-common
 . /usr/sbin/so-elastic-fleet-common
@@ -16,6 +19,7 @@ BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
 BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
 BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
 PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
+COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json
 
 PENDING_UPDATE=false
 
@@ -46,13 +50,43 @@ compare_versions() {
     fi
 }
 
+IFS=$'\n'
+agent_policies=$(elastic_fleet_agent_policy_ids)
+if [ $? -ne 0 ]; then
+    echo "Error: Failed to retrieve agent policies."
+    exit 1
+fi
+
+default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
+
+in_use_integrations=()
+
+for AGENT_POLICY in $agent_policies; do
+
+    if ! integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY"); then
+        # skip the agent policy if we can't get required info, let salt retry. Integrations loaded by this script are non-default integrations.
+        echo "Skipping $AGENT_POLICY.. "
+        continue
+    fi
+    for INTEGRATION in $integrations; do
+        if ! PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION"); then
+            echo  "Not adding $INTEGRATION, couldn't get package name"
+            continue
+        fi
+        # non-default integrations that are in-use in any policy
+        if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
+            in_use_integrations+=("$PACKAGE_NAME")
+        fi
+    done
+done
+
 if [[ -f $STATE_FILE_SUCCESS  ]]; then
     if retry 3 1 "curl -s -K /opt/so/conf/elasticsearch/curl.config --output /dev/null --silent --head --fail localhost:5601/api/fleet/epm/packages"; then
         # Package_list contains all integrations beta / non-beta.
         latest_package_list=$(/usr/sbin/so-elastic-fleet-package-list)
         echo '{ "packages" : []}' > $BULK_INSTALL_PACKAGE_LIST
         rm -f $INSTALLED_PACKAGE_LIST
-        echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .savedObject.attributes.install_version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
+        echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .installationInfo.version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
 
         while read -r package; do
             # get package details
@@ -77,10 +111,19 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
                     else
                         results=$(compare_versions "$latest_version" "$installed_version")
                         if [ $results == "greater" ]; then
-                            echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
-                            jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                            {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations  #}
+                            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                            if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
+                            {%- endif %}
+                                echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
+                                jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
 
-                            PENDING_UPDATE=true
+                                PENDING_UPDATE=true
+                            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                            else
+                                echo "skipping available upgrade for in use integration - $package_name."
+                            fi
+                            {%- endif %}
                         fi
                     fi
                 fi
@@ -92,9 +135,18 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
                 else
                     results=$(compare_versions "$latest_version" "$installed_version")
                     if [ $results == "greater" ]; then
-                        echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
-                        jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
-                        PENDING_UPDATE=true
+                        {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations  #}
+                        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                        if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
+                        {%- endif %}
+                            echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
+                            jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                            PENDING_UPDATE=true
+                        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                        else
+                            echo "skipping available upgrade for in use integration - $package_name."
+                        fi
+                        {%- endif %}
                     fi
                 fi
             {% endif %}
@@ -104,14 +156,38 @@ if [[ -f $STATE_FILE_SUCCESS  ]]; then
         done <<< "$(jq -c '.packages[]' "$INSTALLED_PACKAGE_LIST")"
 
         if [ "$PENDING_UPDATE" = true ]; then
-            # Run bulk install of packages
-            elastic_fleet_bulk_package_install $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_OUTPUT
+            # Run chunked install of packages
+            echo "" > $BULK_INSTALL_OUTPUT
+            pkg_group=1
+            pkg_filename="${BULK_INSTALL_PACKAGE_LIST%.json}"
+
+            jq -c '.packages | _nwise(25)' $BULK_INSTALL_PACKAGE_LIST | while read -r line; do
+                echo "$line" | jq '{ "packages": . }' > "${pkg_filename}_${pkg_group}.json"
+                pkg_group=$((pkg_group + 1))
+            done
+
+            for file in "${pkg_filename}_"*.json; do
+                [ -e "$file" ] || continue
+                if ! elastic_fleet_bulk_package_install $file >> $BULK_INSTALL_OUTPUT; then
+                    # integrations loaded my this script are non-essential and shouldn't cause exit, skip them for now next highstate run can retry
+                    echo "Failed to complete a chunk of bulk package installs -- $file "
+                    continue
+                fi
+            done
+            # cleanup any temp files for chunked package install
+            rm -f ${pkg_filename}_*.json $BULK_INSTALL_PACKAGE_LIST
         else
             echo "Elastic integrations don't appear to need installation/updating..."
         fi
         # Write out file for generating index/component/ilm templates
-        latest_installed_package_list=$(elastic_fleet_installed_packages)
-        echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+        if latest_installed_package_list=$(elastic_fleet_installed_packages); then
+            echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+        fi
+        if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
+            # Refresh installed component template list
+            latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')
+            echo $latest_component_templates_list > $COMPONENT_TEMPLATES
+        fi
 
     else
         # This is the installation of add-on integrations and upgrade of existing integrations. Exiting without error, next highstate will attempt to re-run.

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -3,11 +3,36 @@
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%- from 'vars/globals.map.jinja' import GLOBALS %}
+{%- from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
+{%- from 'elasticfleet/config.map.jinja' import LOGSTASH_CONFIG_YAML %}
 
 . /usr/sbin/so-common
 
+FORCE_UPDATE=false
+UPDATE_CERTS=false
+LOGSTASH_PILLAR_CONFIG_YAML="{{ LOGSTASH_CONFIG_YAML }}"
+LOGSTASH_PILLAR_STATE_FILE="/opt/so/state/esfleet_logstash_config_pillar"
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -f|--force)
+      FORCE_UPDATE=true
+      shift
+      ;;
+    -c| --certs)
+      UPDATE_CERTS=true
+      FORCE_UPDATE=true
+      shift
+      ;;
+    *)
+      echo "Unknown option $1"
+      echo "Usage: $0 [-f|--force] [-c|--certs]"
+      exit 1
+      ;;
+  esac
+done
+
 # Only run on Managers
 if ! is_manager_node; then
     printf "Not a Manager Node... Exiting"
@@ -15,27 +40,109 @@ if ! is_manager_node; then
 fi
 
 function update_logstash_outputs() {
-	# Generate updated JSON payload
-    JSON_STRING=$(jq -n --arg UPDATEDLIST $NEW_LIST_JSON '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":""}')
+    if logstash_policy=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_logstash" --retry 3 --retry-delay 10 --fail 2>/dev/null); then
+        SSL_CONFIG=$(echo "$logstash_policy" | jq -r '.item.ssl')
+        LOGSTASHKEY=$(openssl rsa -in  /etc/pki/elasticfleet-logstash.key)
+        LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt)
+        LOGSTASHCA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
+        # Revert escaped \\n to \n for jq
+        LOGSTASH_PILLAR_CONFIG_YAML=$(printf '%b' "$LOGSTASH_PILLAR_CONFIG_YAML")
+
+        if SECRETS=$(echo "$logstash_policy" | jq -er '.item.secrets' 2>/dev/null); then
+            if [[ "$UPDATE_CERTS" != "true"  ]]; then
+                # Reuse existing secret
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
+                    --argjson SECRETS "$SECRETS" \
+                    --argjson SSL_CONFIG "$SSL_CONFIG" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+            else
+                # Update certs, creating new secret
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
+                    --arg LOGSTASHKEY "$LOGSTASHKEY" \
+                    --arg LOGSTASHCRT "$LOGSTASHCRT" \
+                    --arg LOGSTASHCA "$LOGSTASHCA" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": {"certificate": $LOGSTASHCRT,"certificate_authorities":[ $LOGSTASHCA ]},"secrets": {"ssl":{"key": $LOGSTASHKEY }}}')
+            fi
+        else
+            if [[ "$UPDATE_CERTS" != "true" ]]; then
+                # Reuse existing ssl config
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
+                    --argjson SSL_CONFIG "$SSL_CONFIG" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": $SSL_CONFIG}')
+            else
+                # Update ssl config
+                JSON_STRING=$(jq -n \
+                    --arg UPDATEDLIST "$NEW_LIST_JSON" \
+                    --arg CONFIG_YAML "$LOGSTASH_PILLAR_CONFIG_YAML" \
+                    --arg LOGSTASHKEY "$LOGSTASHKEY" \
+                    --arg LOGSTASHCRT "$LOGSTASHCRT" \
+                    --arg LOGSTASHCA "$LOGSTASHCA" \
+                    '{"name":"grid-logstash","type":"logstash","hosts": $UPDATEDLIST,"is_default":true,"is_default_monitoring":true,"config_yaml":$CONFIG_YAML,"ssl": {"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}')
+            fi
+        fi
+    fi
 
     # Update Logstash Outputs
     curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
 }
 function update_kafka_outputs() {
   # Make sure SSL configuration is included in policy updates for Kafka output. SSL is configured in so-elastic-fleet-setup
-  SSL_CONFIG=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" | jq -r '.item.ssl')
-
-  JSON_STRING=$(jq -n \
-    --arg UPDATEDLIST "$NEW_LIST_JSON" \
-    --argjson SSL_CONFIG "$SSL_CONFIG" \
-    '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}')
-  # Update Kafka outputs
-  curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
+  if kafka_policy=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" --fail 2>/dev/null); then
+    SSL_CONFIG=$(echo "$kafka_policy" | jq -r '.item.ssl')
+    KAFKAKEY=$(openssl rsa -in  /etc/pki/elasticfleet-kafka.key)
+    KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt)
+    KAFKACA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
+    if SECRETS=$(echo "$kafka_policy" | jq -er '.item.secrets' 2>/dev/null); then
+        if [[ "$UPDATE_CERTS" != "true" ]]; then
+            # Update policy when fleet has secrets enabled
+            JSON_STRING=$(jq -n \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
+              --argjson SSL_CONFIG "$SSL_CONFIG" \
+              --argjson SECRETS "$SECRETS" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG,"secrets": $SECRETS}')
+        else
+            # Update certs, creating new secret
+            JSON_STRING=$(jq -n \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
+              --arg KAFKAKEY "$KAFKAKEY" \
+              --arg KAFKACRT "$KAFKACRT" \
+              --arg KAFKACA "$KAFKACA" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": {"certificate_authorities":[ $KAFKACA ],"certificate": $KAFKACRT ,"key":"","verification_mode":"full"},"secrets": {"ssl":{"key": $KAFKAKEY }}}')
+        fi
+    else
+        if [[ "$UPDATE_CERTS" != "true" ]]; then
+            # Update policy when fleet has secrets disabled or policy hasn't been force updated
+            JSON_STRING=$(jq -n \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
+              --argjson SSL_CONFIG "$SSL_CONFIG" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}')
+        else
+            # Update ssl config
+            JSON_STRING=$(jq -n \
+              --arg UPDATEDLIST "$NEW_LIST_JSON" \
+              --arg KAFKAKEY "$KAFKAKEY" \
+              --arg KAFKACRT "$KAFKACRT" \
+              --arg KAFKACA "$KAFKACA" \
+              '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }}')
+        fi
+    fi
+    # Update Kafka outputs
+    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
+  else
+    printf "Failed to get current Kafka output policy..."
+    exit 1
+  fi
 }
 
 {% if GLOBALS.pipeline == "KAFKA" %}
   # Get current list of Kafka Outputs
-  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_kafka')
+  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_kafka' --retry 3 --retry-delay 30 --fail 2>/dev/null)
 
   # Check to make sure that the server responded with good data - else, bail from script
   CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
@@ -46,7 +153,7 @@ function update_kafka_outputs() {
 
   # Get the current list of kafka outputs & hash them
   CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
-  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+  CURRENT_HASH=$(sha256sum <<< "$CURRENT_LIST" | awk '{print $1}')
 
   declare -a NEW_LIST=()
 
@@ -61,7 +168,7 @@ function update_kafka_outputs() {
   {# If global pipeline isn't set to KAFKA then assume default of REDIS / logstash #}
 {% else %}
   # Get current list of Logstash Outputs
-  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash')
+  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash' --retry 3 --retry-delay 30 --fail 2>/dev/null)
 
   # Check to make sure that the server responded with good data - else, bail from script
   CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
@@ -69,10 +176,19 @@ function update_kafka_outputs() {
    printf "Failed to query for current Logstash Outputs..."
    exit 1
   fi
+  # logstash adv config - compare pillar to last state file value
+  if [[ -f "$LOGSTASH_PILLAR_STATE_FILE" ]]; then
+    PREVIOUS_LOGSTASH_PILLAR_CONFIG_YAML=$(cat "$LOGSTASH_PILLAR_STATE_FILE")
+    if [[ "$LOGSTASH_PILLAR_CONFIG_YAML" != "$PREVIOUS_LOGSTASH_PILLAR_CONFIG_YAML" ]]; then
+      echo "Logstash pillar config has changed - forcing update"
+      FORCE_UPDATE=true
+    fi
+    echo "$LOGSTASH_PILLAR_CONFIG_YAML" > "$LOGSTASH_PILLAR_STATE_FILE"
+  fi
 
   # Get the current list of Logstash outputs & hash them
   CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
-  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
+  CURRENT_HASH=$(sha256sum <<< "$CURRENT_LIST" | awk '{print $1}')
 
   declare -a NEW_LIST=()
 
@@ -121,10 +237,10 @@ function update_kafka_outputs() {
 
 # Sort & hash the new list of Logstash Outputs
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
-NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
+NEW_HASH=$(sha256sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
 
 # Compare the current & new list of outputs - if different, update the Logstash outputs
-if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
+if [[ "$NEW_HASH" = "$CURRENT_HASH" ]] && [[ "$FORCE_UPDATE" != "true" ]]; then
     printf "\nHashes match - no update needed.\n"
     printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
 

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-package-load

@@ -10,8 +10,16 @@
 
 {%- for PACKAGE in SUPPORTED_PACKAGES %}
 echo "Setting up {{ PACKAGE }} package..."
-VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}")
-elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"
+if VERSION=$(elastic_fleet_package_version_check "{{ PACKAGE }}"); then
+    if ! elastic_fleet_package_install "{{ PACKAGE }}" "$VERSION"; then
+        # packages loaded by this script should never fail to install and REQUIRED before an installation of SO can be considered successful
+        echo -e "\nERROR: Failed to install default integration package -- $PACKAGE $VERSION"
+        exit 1
+    fi
+else
+    echo -e "\nERROR: Failed to get version information for integration $PACKAGE"
+    exit 1
+fi
 echo
 {%- endfor %}
 echo