Merge pull request #14917 from Security-Onion-Solutions/2.4/dev

2.4.170

.github/.gitleaks.toml

@@ -536,10 +536,11 @@ secretGroup = 4
 
 [allowlist]
 description = "global allow lists"
-regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''', '''ssl_.*password''']
+regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''', '''ssl_.*password''', '''integration_key\s=\s"so-logs-"''']
 paths = [
     '''gitleaks.toml''',
     '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',
     '''(go.mod|go.sum)$''',
-    '''salt/nginx/files/enterprise-attack.json'''
+    '''salt/nginx/files/enterprise-attack.json''',
+    '''(.*?)whl$'''
 ]

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -22,6 +22,14 @@ body:
         - 2.4.90
         - 2.4.100
         - 2.4.110
+        - 2.4.111
+        - 2.4.120
+        - 2.4.130
+        - 2.4.140
+        - 2.4.141
+        - 2.4.150
+        - 2.4.160
+        - 2.4.170
         - Other (please provide detail below)
     validations:
       required: true

.github/ISSUE_TEMPLATE

@@ -1,12 +0,0 @@
-PLEASE STOP AND READ THIS INFORMATION!
-
-If you are creating an issue just to ask a question, you will likely get faster and better responses by posting to our discussions forum instead:
-https://securityonion.net/discuss
-
-If you think you have found a possible bug or are observing a behavior that you weren't expecting, use the discussion forum to start a conversation about it instead of creating an issue.
-
-If you are very familiar with the latest version of the product and are confident you have found a bug in Security Onion, you can continue with creating an issue here, but please make sure you have done the following:
-- duplicated the issue on a fresh installation of the latest version
-- provide information about your system and how you installed Security Onion
-- include relevant log files
-- include reproduction steps

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: This option is for experienced community members to report a confirmed, reproducible bug
+title: ''
+labels: ''
+assignees: ''
+
+---
+PLEASE STOP AND READ THIS INFORMATION!
+
+If you are creating an issue just to ask a question, you will likely get faster and better responses by posting to our discussions forum at https://securityonion.net/discuss.
+
+If you think you have found a possible bug or are observing a behavior that you weren't expecting, use the discussion forum at https://securityonion.net/discuss to start a conversation about it instead of creating an issue.
+
+If you are very familiar with the latest version of the product and are confident you have found a bug in Security Onion, you can continue with creating an issue here, but please make sure you have done the following:
+- duplicated the issue on a fresh installation of the latest version
+- provide information about your system and how you installed Security Onion
+- include relevant log files
+- include reproduction steps
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Security Onion Discussions
+    url: https://securityonion.com/discussions
+    about: Please ask and answer questions here

.github/workflows/contrib.yml

@@ -18,7 +18,7 @@ jobs:
         with:
           path-to-signatures: 'signatures_v1.json'
           path-to-document: 'https://securityonionsolutions.com/cla' 
-          allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,weslambert,defensivedepth,m0duspwnens
+          allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,defensivedepth,m0duspwnens
           remote-organization-name: Security-Onion-Solutions
           remote-repository-name: licensing
 

.github/workflows/pythontest.yml

@@ -1,10 +1,6 @@
 name: python-test
 
 on:
-  push:
-    paths: 
-      - "salt/sensoroni/files/analyzers/**"
-      - "salt/manager/tools/sbin"
   pull_request:
     paths:
       - "salt/sensoroni/files/analyzers/**"
@@ -17,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.13"]
         python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"]
 
     steps:
@@ -36,4 +32,4 @@ jobs:
         flake8 ${{ matrix.python-code-path }} --show-source --max-complexity=12  --doctests --max-line-length=200 --statistics
     - name: Test with pytest
       run: |
-        pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini
+        PYTHONPATH=${{ matrix.python-code-path }} pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini

.gitignore

@@ -1,4 +1,3 @@
-
 # Created by https://www.gitignore.io/api/macos,windows
 # Edit at https://www.gitignore.io/?templates=macos,windows
 
@@ -67,4 +66,4 @@ __pycache__
 
 # Analyzer dev/test config files
 *_dev.yaml
-site-packages
+site-packages

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.111-20241217 ISO image released on 2024/12/18
+### 2.4.170-20250812 ISO image released on 2025/08/12
 
 
 ### Download and Verify
 
-2.4.111-20241217 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.111-20241217.iso
+2.4.170-20250812 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.170-20250812.iso
  
-MD5: 767823D75EB76A6DC6132F799FD0E720  
-SHA1: 0A7B6918FE5D4BC89EE3F2E03B4F8F4D6255141D  
-SHA256: 394BFCED9B5EAA0788E2D04806231B3A170839394AAF8DD23B4CE0EB9D6EF727  
+MD5: 50ECAAD05736298452DECEAE074FA773  
+SHA1: 1B1EB520DE61ECC4BF34E512DAFE307317D7666A  
+SHA256: 87D176A48A58BAD1C2D57196F999BED23DE9B526226E3754F0C166C866CCDC1A  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.111-20241217.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.170-20250812.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.111-20241217.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.170-20250812.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.111-20241217.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.170-20250812.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.111-20241217.iso.sig securityonion-2.4.111-20241217.iso
+gpg --verify securityonion-2.4.170-20250812.iso.sig securityonion-2.4.170-20250812.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Tue 17 Dec 2024 04:33:10 PM EST using RSA key ID FE507013
+gpg: Signature made Fri 08 Aug 2025 06:24:56 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

HOTFIX

@@ -1 +0,0 @@
-

LICENSE

@@ -0,0 +1,53 @@
+Elastic License 2.0 (ELv2)
+
+Acceptance
+
+  By using the software, you agree to all of the terms and conditions below.
+
+Copyright License
+
+  The licensor grants you a non-exclusive, royalty-free, worldwide, non-sublicensable, non-transferable license to use, copy, distribute, make available, and prepare derivative works of the software, in each case subject to the limitations and conditions below.
+
+Limitations
+
+  You may not provide the software to third parties as a hosted or managed service, where the service provides users with access to any substantial set of the features or functionality of the software.
+
+  You may not move, change, disable, or circumvent the license key functionality in the software, and you may not remove or obscure any functionality in the software that is protected by the license key.
+
+  You may not alter, remove, or obscure any licensing, copyright, or other notices of the licensor in the software. Any use of the licensor’s trademarks is subject to applicable law.
+
+Patents
+
+  The licensor grants you a license, under any patent claims the licensor can license, or becomes able to license, to make, have made, use, sell, offer for sale, import and have imported the software, in each case subject to the limitations and conditions in this license. This license does not cover any patent claims that you cause to be infringed by modifications or additions to the software. If you or your company make any written claim that the software infringes or contributes to infringement of any patent, your patent license for the software granted under these terms ends immediately. If your company makes such a claim, your patent license ends immediately for work on behalf of your company.
+
+Notices
+
+  You must ensure that anyone who gets a copy of any part of the software from you also gets a copy of these terms.
+
+  If you modify the software, you must include in any modified copies of the software prominent notices stating that you have modified the software.
+
+No Other Rights
+
+  These terms do not imply any licenses other than those expressly granted in these terms.
+
+Termination
+
+  If you use the software in violation of these terms, such use is not licensed, and your licenses will automatically terminate. If the licensor provides you with a notice of your violation, and you cease all violation of this license no later than 30 days after you receive that notice, your licenses will be reinstated retroactively. However, if you violate these terms after such reinstatement, any additional violation of these terms will cause your licenses to terminate automatically and permanently.
+
+No Liability
+
+  As far as the law allows, the software comes as is, without any warranty or condition, and the licensor will not be liable to you for any damages arising out of these terms or the use or nature of the software, under any kind of legal claim.
+
+Definitions
+
+  The licensor is the entity offering these terms, and the software is the software the licensor makes available under these terms, including any portion of it.
+
+  you refers to the individual or entity agreeing to these terms.
+
+  your company is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization. control means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise. Control can be direct or indirect.
+
+  your licenses are all the licenses granted to you for the software under these terms.
+
+  use means anything you do with the software requiring one of your licenses.
+  
+  trademark means trademarks, service marks, and similar rights.

VERSION

@@ -1 +1 @@
-2.4.111
+2.4.170

pillar/hypervisor/nodes.sls

@@ -0,0 +1,34 @@
+{% set node_types = {} %}
+{% for minionid, ip in salt.saltutil.runner(
+    'mine.get',
+    tgt='G@role:so-hypervisor or G@role:so-managerhype',
+    fun='network.ip_addrs',
+    tgt_type='compound') | dictsort()
+%}
+
+# only add a node to the pillar if it returned an ip from the mine
+{%   if ip | length > 0%}
+{%     set hostname = minionid.split('_') | first %}
+{%     set node_type = minionid.split('_') | last %}
+{%     if node_type not in node_types.keys() %}
+{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
+{%     else %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: ip[0]}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update(ip[0]) %}
+{%       endif %}
+{%     endif %}
+{%   endif %}
+{% endfor %}
+
+
+hypervisor:
+  nodes:
+{% for node_type, values in node_types.items() %}
+    {{node_type}}:
+{%   for hostname, ip in values.items() %}
+      {{hostname}}:
+        ip: {{ip}}
+{%   endfor %}
+{% endfor %}

pillar/node_data/ips.sls

@@ -24,6 +24,7 @@
 {%   endif %}
 {% endfor %}
 
+{% if node_types %}
 node_data:
 {% for node_type, host_values in node_types.items() %}
 {%   for hostname, details in host_values.items() %}
@@ -33,3 +34,6 @@ node_data:
     role: {{node_type}}
 {%   endfor %}
 {% endfor %}
+{% else %}
+node_data: False
+{% endif %}

pillar/top.sls

@@ -16,16 +16,24 @@ base:
     - sensoroni.adv_sensoroni
     - telegraf.soc_telegraf
     - telegraf.adv_telegraf
+    - versionlock.soc_versionlock
+    - versionlock.adv_versionlock
+    - soc.license
 
   '* and not *_desktop':
     - firewall.soc_firewall
     - firewall.adv_firewall
     - nginx.soc_nginx
     - nginx.adv_nginx
-    - node_data.ips
 
-  '*_manager or *_managersearch':
+  'salt-cloud:driver:libvirt':
+    - match: grain
+    - vm.soc_vm
+    - vm.adv_vm
+
+  '*_manager or *_managersearch or *_managerhype':
     - match: compound
+    - node_data.ips
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
@@ -42,11 +50,12 @@ base:
     - logstash.adv_logstash
     - soc.soc_soc
     - soc.adv_soc
-    - soc.license
     - kibana.soc_kibana
     - kibana.adv_kibana
     - kratos.soc_kratos
     - kratos.adv_kratos
+    - hydra.soc_hydra
+    - hydra.adv_hydra
     - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
@@ -66,6 +75,9 @@ base:
     - kafka.nodes
     - kafka.soc_kafka
     - kafka.adv_kafka
+    - hypervisor.nodes
+    - hypervisor.soc_hypervisor
+    - hypervisor.adv_hypervisor
     - stig.soc_stig
 
   '*_sensor':
@@ -83,9 +95,9 @@ base:
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
-    - soc.license
 
   '*_eval':
+    - node_data.ips
     - secrets
     - healthcheck.eval
     - elasticsearch.index_templates
@@ -96,6 +108,7 @@ base:
     - kibana.secrets
     {% endif %}
     - kratos.soc_kratos
+    - kratos.adv_kratos
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     - elasticfleet.soc_elasticfleet
@@ -108,13 +121,12 @@ base:
     - idstools.adv_idstools
     - soc.soc_soc
     - soc.adv_soc
-    - soc.license
     - kibana.soc_kibana
     - kibana.adv_kibana
     - strelka.soc_strelka
     - strelka.adv_strelka
-    - kratos.soc_kratos
-    - kratos.adv_kratos
+    - hydra.soc_hydra
+    - hydra.adv_hydra
     - redis.soc_redis
     - redis.adv_redis
     - influxdb.soc_influxdb
@@ -133,6 +145,7 @@ base:
     - minions.adv_{{ grains.id }}
 
   '*_standalone':
+    - node_data.ips
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
@@ -149,6 +162,8 @@ base:
     - idstools.adv_idstools
     - kratos.soc_kratos
     - kratos.adv_kratos
+    - hydra.soc_hydra
+    - hydra.adv_hydra
     - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
@@ -165,7 +180,6 @@ base:
     - manager.adv_manager
     - soc.soc_soc
     - soc.adv_soc
-    - soc.license
     - kibana.soc_kibana
     - kibana.adv_kibana
     - strelka.soc_strelka
@@ -231,7 +245,6 @@ base:
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
-    - soc.license
     - kafka.nodes
     - kafka.soc_kafka
     - kafka.adv_kafka
@@ -249,10 +262,9 @@ base:
     - minions.adv_{{ grains.id }}
     - kafka.nodes
     - kafka.soc_kafka
-    - kafka.adv_kafka
-    - soc.license
 
   '*_import':
+    - node_data.ips
     - secrets
     - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
@@ -262,6 +274,7 @@ base:
     - kibana.secrets
     {% endif %}
     - kratos.soc_kratos
+    - kratos.adv_kratos
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     - elasticfleet.soc_elasticfleet
@@ -272,13 +285,12 @@ base:
     - manager.adv_manager
     - soc.soc_soc
     - soc.adv_soc
-    - soc.license
     - kibana.soc_kibana
     - kibana.adv_kibana
     - backup.soc_backup
     - backup.adv_backup
-    - kratos.soc_kratos
-    - kratos.adv_kratos
+    - hydra.soc_hydra
+    - hydra.adv_hydra
     - redis.soc_redis
     - redis.adv_redis
     - influxdb.soc_influxdb
@@ -297,6 +309,7 @@ base:
     - minions.adv_{{ grains.id }}
 
   '*_fleet':
+    - node_data.ips
     - backup.soc_backup
     - backup.adv_backup
     - logstash.nodes
@@ -307,8 +320,12 @@ base:
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
+  '*_hypervisor':
+    - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
+
   '*_desktop':
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
     - stig.soc_stig
-    - soc.license
+

salt/_modules/qcow2.py

@@ -0,0 +1,246 @@
+#!py
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+"""
+Salt module for managing QCOW2 image configurations and VM hardware settings. This module provides functions
+for modifying network configurations within QCOW2 images and adjusting virtual machine hardware settings.
+It serves as a Salt interface to the so-qcow2-modify-network and so-kvm-modify-hardware scripts.
+
+The module offers two main capabilities:
+1. Network Configuration: Modify network settings (DHCP/static IP) within QCOW2 images
+2. Hardware Configuration: Adjust VM hardware settings (CPU, memory, PCI passthrough)
+
+This module is intended to work with Security Onion's virtualization infrastructure and is typically
+used in conjunction with salt-cloud for VM provisioning and management.
+"""
+
+import logging
+import subprocess
+import shlex
+
+log = logging.getLogger(__name__)
+
+__virtualname__ = 'qcow2'
+
+def __virtual__():
+    return __virtualname__
+
+def modify_network_config(image, interface, mode, vm_name, ip4=None, gw4=None, dns4=None, search4=None):
+    '''
+    Usage:
+        salt '*' qcow2.modify_network_config image=<path> interface=<iface> mode=<mode> vm_name=<name> [ip4=<addr>] [gw4=<addr>] [dns4=<servers>] [search4=<domain>]
+
+    Options:
+        image
+            Path to the QCOW2 image file that will be modified
+        interface
+            Network interface name to configure (e.g., 'enp1s0')
+        mode
+            Network configuration mode, either 'dhcp4' or 'static4'
+        vm_name
+            Full name of the VM (hostname_role)
+        ip4
+            IPv4 address with CIDR notation (e.g., '192.168.1.10/24')
+            Required when mode='static4'
+        gw4
+            IPv4 gateway address (e.g., '192.168.1.1')
+            Required when mode='static4'
+        dns4
+            Comma-separated list of IPv4 DNS servers (e.g., '8.8.8.8,8.8.4.4')
+            Optional for both DHCP and static configurations
+        search4
+            DNS search domain for IPv4 (e.g., 'example.local')
+            Optional for both DHCP and static configurations
+
+    Examples:
+        1. **Configure DHCP:**
+            ```bash
+            salt '*' qcow2.modify_network_config image='/nsm/libvirt/images/sool9/sool9.qcow2' interface='enp1s0' mode='dhcp4'
+            ```
+            This configures enp1s0 to use DHCP for IP assignment
+
+        2. **Configure Static IP:**
+            ```bash
+            salt '*' qcow2.modify_network_config image='/nsm/libvirt/images/sool9/sool9.qcow2' interface='enp1s0' mode='static4' ip4='192.168.1.10/24' gw4='192.168.1.1' dns4='192.168.1.1,8.8.8.8' search4='example.local'
+            ```
+            This sets a static IP configuration with DNS servers and search domain
+
+    Notes:
+        - The QCOW2 image must be accessible and writable by the salt minion
+        - The image should not be in use by a running VM when modified
+        - Network changes take effect on next VM boot
+        - Requires so-qcow2-modify-network script to be installed
+
+    Description:
+        This function modifies network configuration within a QCOW2 image file by executing
+        the so-qcow2-modify-network script. It supports both DHCP and static IPv4 configuration.
+        The script mounts the image, modifies the network configuration files, and unmounts
+        safely. All operations are logged for troubleshooting purposes.
+
+    Exit Codes:
+        0: Success
+        1: Invalid parameters or configuration
+        2: Image access or mounting error
+        3: Network configuration error
+        4: System command error
+        255: Unexpected error
+
+    Logging:
+        - All operations are logged to the salt minion log
+        - Log entries are prefixed with 'qcow2 module:'
+        - Error conditions include detailed error messages and stack traces
+        - Success/failure status is logged for verification
+    '''
+
+    cmd = ['/usr/sbin/so-qcow2-modify-network', '-I', image, '-i', interface, '-n', vm_name]
+
+    if mode.lower() == 'dhcp4':
+        cmd.append('--dhcp4')
+    elif mode.lower() == 'static4':
+        cmd.append('--static4')
+        if not ip4 or not gw4:
+            raise ValueError('Both ip4 and gw4 are required for static configuration.')
+        cmd.extend(['--ip4', ip4, '--gw4', gw4])
+        if dns4:
+            cmd.extend(['--dns4', dns4])
+        if search4:
+            cmd.extend(['--search4', search4])
+    else:
+        raise ValueError("Invalid mode '{}'. Expected 'dhcp4' or 'static4'.".format(mode))
+
+    log.info('qcow2 module: Executing command: {}'.format(' '.join(shlex.quote(arg) for arg in cmd)))
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+        ret = {
+            'retcode': result.returncode,
+            'stdout': result.stdout,
+            'stderr': result.stderr
+        }
+        if result.returncode != 0:
+            log.error('qcow2 module: Script execution failed with return code {}: {}'.format(result.returncode, result.stderr))
+        else:
+            log.info('qcow2 module: Script executed successfully.')
+        return ret
+    except Exception as e:
+        log.error('qcow2 module: An error occurred while executing the script: {}'.format(e))
+        raise
+
+def modify_hardware_config(vm_name, cpu=None, memory=None, pci=None, start=False):
+    '''
+    Usage:
+        salt '*' qcow2.modify_hardware_config vm_name=<name> [cpu=<count>] [memory=<size>] [pci=<id>] [pci=<id>] [start=<bool>]
+
+    Options:
+        vm_name
+            Name of the virtual machine to modify
+        cpu
+            Number of virtual CPUs to assign (positive integer)
+            Optional - VM's current CPU count retained if not specified
+        memory
+            Amount of memory to assign in MiB (positive integer)
+            Optional - VM's current memory size retained if not specified
+        pci
+            PCI hardware ID(s) to passthrough to the VM (e.g., '0000:c7:00.0')
+            Can be specified multiple times for multiple devices
+            Optional - no PCI passthrough if not specified
+        start
+            Boolean flag to start the VM after modification
+            Optional - defaults to False
+
+    Examples:
+        1. **Modify CPU and Memory:**
+            ```bash
+            salt '*' qcow2.modify_hardware_config vm_name='sensor1' cpu=4 memory=8192
+            ```
+            This assigns 4 CPUs and 8GB memory to the VM
+
+        2. **Enable PCI Passthrough:**
+            ```bash
+            salt '*' qcow2.modify_hardware_config vm_name='sensor1' pci='0000:c7:00.0' pci='0000:c4:00.0' start=True
+            ```
+            This configures PCI passthrough and starts the VM
+
+        3. **Complete Hardware Configuration:**
+            ```bash
+            salt '*' qcow2.modify_hardware_config vm_name='sensor1' cpu=8 memory=16384 pci='0000:c7:00.0' start=True
+            ```
+            This sets CPU, memory, PCI passthrough, and starts the VM
+
+    Notes:
+        - VM must be stopped before modification unless only the start flag is set
+        - Memory is specified in MiB (1024 = 1GB)
+        - PCI devices must be available and not in use by the host
+        - CPU count should align with host capabilities
+        - Requires so-kvm-modify-hardware script to be installed
+
+    Description:
+        This function modifies the hardware configuration of a KVM virtual machine using
+        the so-kvm-modify-hardware script. It can adjust CPU count, memory allocation,
+        and PCI device passthrough. Changes are applied to the VM's libvirt configuration.
+        The VM can optionally be started after modifications are complete.
+
+    Exit Codes:
+        0: Success
+        1: Invalid parameters
+        2: VM state error (running when should be stopped)
+        3: Hardware configuration error
+        4: System command error
+        255: Unexpected error
+
+    Logging:
+        - All operations are logged to the salt minion log
+        - Log entries are prefixed with 'qcow2 module:'
+        - Hardware configuration changes are logged
+        - Errors include detailed messages and stack traces
+        - Final status of modification is logged
+    '''
+
+    cmd = ['/usr/sbin/so-kvm-modify-hardware', '-v', vm_name]
+
+    if cpu is not None:
+        if isinstance(cpu, int) and cpu > 0:
+            cmd.extend(['-c', str(cpu)])
+        else:
+            raise ValueError('cpu must be a positive integer.')
+    if memory is not None:
+        if isinstance(memory, int) and memory > 0:
+            cmd.extend(['-m', str(memory)])
+        else:
+            raise ValueError('memory must be a positive integer.')
+    if pci:
+        # Handle PCI IDs (can be a single device or comma-separated list)
+        if isinstance(pci, str):
+            devices = [dev.strip() for dev in pci.split(',') if dev.strip()]
+        elif isinstance(pci, list):
+            devices = pci
+        else:
+            devices = [pci]
+
+        # Add each device with its own -p flag
+        for device in devices:
+            cmd.extend(['-p', str(device)])
+    if start:
+        cmd.append('-s')
+
+    log.info('qcow2 module: Executing command: {}'.format(' '.join(shlex.quote(arg) for arg in cmd)))
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+        ret = {
+            'retcode': result.returncode,
+            'stdout': result.stdout,
+            'stderr': result.stderr
+        }
+        if result.returncode != 0:
+            log.error('qcow2 module: Script execution failed with return code {}: {}'.format(result.returncode, result.stderr))
+        else:
+            log.info('qcow2 module: Script executed successfully.')
+        return ret
+    except Exception as e:
+        log.error('qcow2 module: An error occurred while executing the script: {}'.format(e))
+        raise

salt/allowed_states.map.jinja

@@ -1,259 +1,179 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+   https://securityonion.net/license; you may not use this file except in compliance with the
+   Elastic License 2.0. #}
 
 {% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %}
 {% import_yaml 'salt/minion.defaults.yaml' as saltversion %}
 {% set saltversion = saltversion.salt.minion.version %}
 
-{# this is the list we are returning from this map file, it gets built below #}
-{% set allowed_states= [] %}
+{# Define common state groups to reduce redundancy #}
+{% set base_states = [
+    'common',
+    'patch.os.schedule',
+    'motd',
+    'salt.minion-check',
+    'sensoroni',
+    'salt.lasthighstate',
+    'salt.minion'
+] %}
+
+{% set ssl_states = [
+    'ssl',
+    'telegraf',
+    'firewall',
+    'schedule',
+    'docker_clean'
+] %}
+
+{% set manager_states = [
+    'salt.master',
+    'ca',
+    'registry',
+    'manager',
+    'nginx',
+    'influxdb',
+    'soc',
+    'kratos',
+    'hydra',
+    'elasticfleet',
+    'elastic-fleet-package-registry',
+    'idstools',
+    'suricata.manager',
+    'utility'
+] %}
+
+{% set sensor_states = [
+    'pcap',
+    'suricata',
+    'healthcheck',
+    'tcpreplay',
+    'zeek',
+    'strelka'
+] %}
+
+{% set kafka_states = [
+    'kafka'
+] %}
+
+{% set stig_states = [
+    'stig'
+] %}
+
+{% set elastic_stack_states = [
+    'elasticsearch',
+    'elasticsearch.auth',
+    'kibana',
+    'kibana.secrets',
+    'elastalert',
+    'logstash',
+    'redis'
+] %}
+
+{# Initialize the allowed_states list #}
+{% set allowed_states = [] %}
 
 {% if grains.saltversion | string == saltversion | string %}
+    {# Map role-specific states #}
+    {% set role_states = {
+        'so-eval': (
+            ssl_states +
+            manager_states +
+            sensor_states +
+            elastic_stack_states | reject('equalto', 'logstash') | list
+        ),
+        'so-heavynode': (
+            ssl_states +
+            sensor_states +
+            ['elasticagent', 'elasticsearch', 'logstash', 'redis', 'nginx']
+        ),
+        'so-idh': (
+            ssl_states +
+            ['idh']
+        ),
+        'so-import': (
+            ssl_states +
+            manager_states +
+            sensor_states | reject('equalto', 'strelka') | reject('equalto', 'healthcheck') | list +
+            ['elasticsearch', 'elasticsearch.auth', 'kibana', 'kibana.secrets', 'strelka.manager']
+        ),
+        'so-manager': (
+            ssl_states +
+            manager_states +
+            ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users', 'strelka.manager'] +
+            stig_states +
+            kafka_states +
+            elastic_stack_states
+        ),
+        'so-managerhype': (
+            ssl_states +
+            manager_states +
+            ['salt.cloud', 'strelka.manager', 'hypervisor', 'libvirt'] +
+            stig_states +
+            kafka_states +
+            elastic_stack_states
+        ),
+        'so-managersearch': (
+            ssl_states +
+            manager_states +
+            ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users', 'strelka.manager'] +
+            stig_states +
+            kafka_states +
+            elastic_stack_states
+        ),
+        'so-searchnode': (
+            ssl_states +
+            ['kafka.ca', 'kafka.ssl', 'elasticsearch', 'logstash', 'nginx'] +
+            stig_states
+        ),
+        'so-standalone': (
+            ssl_states +
+            manager_states +
+            ['salt.cloud', 'libvirt.packages', 'libvirt.ssh.users'] +
+            sensor_states +
+            stig_states +
+            kafka_states +
+            elastic_stack_states
+        ),
+        'so-sensor': (
+            ssl_states +
+            sensor_states +
+            ['nginx'] +
+            stig_states
+        ),
+        'so-fleet': (
+            ssl_states +
+            ['logstash', 'nginx', 'healthcheck', 'elasticfleet']
+        ),
+        'so-receiver': (
+            ssl_states +
+            kafka_states +
+            stig_states +
+            ['logstash', 'redis']
+        ),
+        'so-hypervisor': (
+            ssl_states +
+            stig_states +
+            ['hypervisor', 'libvirt']
+        ),
+        'so-desktop': (
+            ['ssl', 'docker_clean', 'telegraf'] +
+            stig_states
+        )
+    } %}
 
-  {% set allowed_states= salt['grains.filter_by']({
-      'so-eval': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'manager',
-          'nginx',
-          'telegraf',
-          'influxdb',
-          'soc',
-          'kratos',
-          'elasticfleet',
-          'elastic-fleet-package-registry',
-          'firewall',
-          'idstools',
-          'suricata.manager',
-          'healthcheck',
-          'pcap',
-          'suricata',
-          'utility',
-          'schedule',
-          'tcpreplay',
-          'docker_clean'
-          ],
-      'so-heavynode': [
-          'ssl',
-          'nginx',
-          'telegraf',
-          'firewall',
-          'pcap',
-          'suricata',
-          'healthcheck',
-          'elasticagent',
-          'schedule',
-          'tcpreplay',
-          'docker_clean'
-          ],
-     'so-idh': [
-          'ssl',
-          'telegraf',
-          'firewall',
-          'idh',
-          'schedule',
-          'docker_clean'
-          ],
-      'so-import': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'manager',
-          'nginx',
-          'strelka.manager',
-          'soc',
-          'kratos',
-          'influxdb',
-          'telegraf',
-          'firewall',
-          'idstools',
-          'suricata.manager',
-          'pcap',
-          'utility',
-          'suricata',
-          'zeek',
-          'schedule',
-          'tcpreplay',
-          'docker_clean',
-          'elasticfleet',
-          'elastic-fleet-package-registry'
-          ],
-      'so-manager': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'manager',
-          'nginx',
-          'telegraf',
-          'influxdb',
-          'strelka.manager',
-          'soc',
-          'kratos',
-          'elasticfleet',
-          'elastic-fleet-package-registry',
-          'firewall',
-          'idstools',
-          'suricata.manager',
-          'utility',
-          'schedule',
-          'docker_clean',
-          'stig',
-          'kafka'
-          ],
-      'so-managersearch': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'nginx',
-          'telegraf',
-          'influxdb',
-          'strelka.manager',
-          'soc',
-          'kratos',
-          'elastic-fleet-package-registry',
-          'elasticfleet',
-          'firewall',
-          'manager',
-          'idstools',
-          'suricata.manager',
-          'utility',
-          'schedule',
-          'docker_clean',
-          'stig',
-          'kafka'
-          ],
-      'so-searchnode': [
-          'ssl',
-          'nginx',
-          'telegraf',
-          'firewall',
-          'schedule',
-          'docker_clean',
-          'stig',
-          'kafka.ca',
-          'kafka.ssl'
-          ],
-      'so-standalone': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'manager',
-          'nginx',
-          'telegraf',
-          'influxdb',
-          'soc',
-          'kratos',
-          'elastic-fleet-package-registry',
-          'elasticfleet',
-          'firewall',
-          'idstools',
-          'suricata.manager',
-          'pcap',
-          'suricata',
-          'healthcheck',
-          'utility',
-          'schedule',
-          'tcpreplay',
-          'docker_clean',
-          'stig',
-          'kafka'
-          ],
-      'so-sensor': [
-          'ssl',
-          'telegraf',
-          'firewall',
-          'nginx',
-          'pcap',
-          'suricata',
-          'healthcheck',
-          'schedule',
-          'tcpreplay',
-          'docker_clean',
-          'stig'
-          ],
-      'so-fleet': [
-          'ssl',
-          'telegraf',
-          'firewall',
-          'logstash',
-          'nginx',
-          'healthcheck',
-          'schedule',
-          'elasticfleet',
-          'docker_clean'
-          ],
-      'so-receiver': [
-          'ssl',
-          'telegraf',
-          'firewall',
-          'schedule',
-          'docker_clean',
-          'kafka',
-          'stig'
-          ],
-      'so-desktop': [
-          'ssl',
-          'docker_clean',
-          'telegraf',
-          'stig'
-          ],
-  }, grain='role') %}
-
-  {%- if grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
-    {% do allowed_states.append('zeek') %}
-  {%- endif %}
-
-  {% if grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
-    {% do allowed_states.append('strelka') %}
-  {% endif %}
-
-  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import'] %}
-    {% do allowed_states.append('elasticsearch') %}
-  {% endif %}
-
-  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
-    {% do allowed_states.append('elasticsearch.auth') %}
-  {% endif %}
-
-  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
-    {% do allowed_states.append('kibana') %}
-    {% do allowed_states.append('kibana.secrets') %}
-  {% endif %}
-
-  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
-    {% do allowed_states.append('elastalert') %}
-  {% endif %}
-
-  {% if grains.role in ['so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
-    {% do allowed_states.append('logstash') %}
-  {% endif %}
-
-  {% if grains.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-receiver', 'so-eval'] %}
-    {% do allowed_states.append('redis') %}
-  {% endif %}
-  
-  {# all nodes on the right salt version can run the following states #}
-  {% do allowed_states.append('common') %}
-  {% do allowed_states.append('patch.os.schedule') %}
-  {% do allowed_states.append('motd') %}
-  {% do allowed_states.append('salt.minion-check') %}
-  {% do allowed_states.append('sensoroni') %}
-  {% do allowed_states.append('salt.lasthighstate') %}
+    {# Get states for the current role #}
+    {% if grains.role in role_states %}
+        {% set allowed_states = role_states[grains.role] %}
+    {% endif %}
 
+    {# Add base states that apply to all roles #}
+    {% for state in base_states %}
+        {% do allowed_states.append(state) %}
+    {% endfor %}
 {% endif %}
 
-
+{# Add airgap state if needed #}
 {% if ISAIRGAP %}
-  {% do allowed_states.append('airgap') %}
+    {% do allowed_states.append('airgap') %}
 {% endif %}
-
-{# all nodes can always run salt.minion state #}
-{% do allowed_states.append('salt.minion') %}

salt/backup/defaults.yaml

@@ -4,4 +4,5 @@ backup:
     - /etc/pki
     - /etc/salt
     - /nsm/kratos
+    - /nsm/hydra
   destination: "/nsm/backup"

salt/backup/tools/sbin/so-config-backup.jinja

@@ -11,6 +11,10 @@ TODAY=$(date '+%Y_%m_%d')
 BACKUPDIR={{ DESTINATION }}
 BACKUPFILE="$BACKUPDIR/so-config-backup-$TODAY.tar"
 MAXBACKUPS=7
+EXCLUSIONS=(
+  "--exclude=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers"
+)
+
 
 # Create backup dir if it does not exist
 mkdir -p /nsm/backup
@@ -23,7 +27,7 @@ if [ ! -f $BACKUPFILE ]; then
 
   # Loop through all paths defined in global.sls, and append them to backup file
   {%- for LOCATION in BACKUPLOCATIONS %}
-  tar -rf $BACKUPFILE {{ LOCATION }}
+  tar -rf $BACKUPFILE "${EXCLUSIONS[@]}" {{ LOCATION }}
   {%- endfor %}
 
 fi

salt/common/init.sls

@@ -106,7 +106,7 @@ Etc/UTC:
   timezone.system
 
 # Sync curl configuration for Elasticsearch authentication
-{% if GLOBALS.role in ['so-eval', 'so-heavynode', 'so-import', 'so-manager', 'so-managersearch', 'so-searchnode', 'so-standalone'] %}
+{% if GLOBALS.is_manager or GLOBALS.role in ['so-heavynode', 'so-searchnode'] %}
 elastic_curl_config:
   file.managed:
     - name: /opt/so/conf/elasticsearch/curl.config
@@ -128,6 +128,11 @@ common_sbin:
     - user: 939
     - group: 939
     - file_mode: 755
+    - show_changes: False
+{% if GLOBALS.role == 'so-heavynode' %}
+    - exclude_pat:
+      - so-pcap-import
+{% endif %}
 
 common_sbin_jinja:
   file.recurse:
@@ -137,6 +142,21 @@ common_sbin_jinja:
     - group: 939 
     - file_mode: 755
     - template: jinja
+    - show_changes: False
+{% if GLOBALS.role == 'so-heavynode' %}
+    - exclude_pat:
+      - so-import-pcap
+{% endif %}
+
+{% if GLOBALS.role == 'so-heavynode' %}
+remove_so-pcap-import_heavynode:
+  file.absent:
+    - name: /usr/sbin/so-pcap-import
+
+remove_so-import-pcap_heavynode:
+  file.absent:
+    - name: /usr/sbin/so-import-pcap
+{% endif %}
 
 {% if not GLOBALS.is_manager%}
 # prior to 2.4.50 these scripts were in common/tools/sbin on the manager because of soup and distributed to non managers
@@ -182,6 +202,7 @@ sostatus_log:
   file.managed:
     - name: /opt/so/log/sostatus/status.log
     - mode: 644
+    - replace: False
 
 # Install sostatus check cron. This is used to populate Grid.
 so-status_check_cron:

salt/common/packages.sls

@@ -1,6 +1,6 @@
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
-{% if GLOBALS.os_family == 'Debian' %}
+# we cannot import GLOBALS from vars/globals.map.jinja in this state since it is called in setup.virt.init
+# since it is early in setup of a new VM, the pillars imported in GLOBALS are not yet defined
+{% if grains.os_family == 'Debian' %}
 commonpkgs:
   pkg.installed:
     - skip_suggestions: True
@@ -27,6 +27,7 @@ commonpkgs:
       - vim
       - tar
       - unzip
+      - bc
       {% if grains.oscodename != 'focal' %}
       - python3-rich
       {% endif %}
@@ -45,7 +46,7 @@ python-rich:
 {%     endif %}
 {% endif %}
 
-{% if GLOBALS.os_family == 'RedHat' %}
+{% if grains.os_family == 'RedHat' %}
 
 remove_mariadb:
   pkg.removed:
@@ -56,6 +57,7 @@ commonpkgs:
     - skip_suggestions: True
     - pkgs:
       - python3-dnf-plugin-versionlock
+      - bc
       - curl
       - device-mapper-persistent-data
       - fuse

salt/common/soup_scripts.sls

@@ -11,6 +11,7 @@
 {%   else %}
 {%     set UPDATE_DIR='/tmp/sogh/securityonion' %}
 {%   endif %}
+{%   set SOVERSION = salt['file.read']('/etc/soversion').strip() %}
 
 remove_common_soup:
   file.absent:
@@ -63,6 +64,12 @@ copy_so-repo-sync_manager_tools_sbin:
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
     - preserve: True
 
+copy_bootstrap-salt_manager_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/salt/scripts/bootstrap-salt.sh
+    - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
+    - preserve: True
+
 # This section is used to put the new script in place so that it can be called during soup.
 # It is faster than calling the states that normally manage them to put them in place.
 copy_so-common_sbin:
@@ -107,6 +114,24 @@ copy_so-repo-sync_sbin:
     - force: True
     - preserve: True
 
+copy_bootstrap-salt_sbin:
+  file.copy:
+    - name: /usr/sbin/bootstrap-salt.sh
+    - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
+    - force: True
+    - preserve: True
+
+{# this is added in 2.4.120 to remove salt repo files pointing to saltproject.io to accomodate the move to broadcom and new bootstrap-salt script #}
+{%   if salt['pkg.version_cmp'](SOVERSION, '2.4.120') == -1 %}
+{%     set saltrepofile = '/etc/yum.repos.d/salt.repo' %}
+{%     if grains.os_family == 'Debian' %}
+{%       set saltrepofile = '/etc/apt/sources.list.d/salt.list' %}
+{%     endif %}
+remove_saltproject_io_repo_manager:
+  file.absent:
+    - name: {{ saltrepofile }}
+{%   endif %}
+
 {% else %}
 fix_23_soup_sbin:
   cmd.run:

salt/common/tools/sbin/so-common

@@ -99,6 +99,17 @@ add_interface_bond0() {
 	fi
 }
 
+airgap_playbooks() {
+	SRC_DIR=$1
+	# Copy playbooks if using airgap
+	mkdir -p /nsm/airgap-resources
+	# Purge old airgap playbooks to ensure SO only uses the latest released playbooks
+	rm -fr /nsm/airgap-resources/playbooks
+	tar xf $SRC_DIR/airgap-resources/playbooks.tgz -C /nsm/airgap-resources/
+	chown -R socore:socore /nsm/airgap-resources/playbooks
+	git config --global --add safe.directory /nsm/airgap-resources/playbooks
+}
+
 check_container() {
 	docker ps | grep "$1:" > /dev/null 2>&1
 	return $?
@@ -226,7 +237,7 @@ create_local_directories() {
 		for d in $(find $PILLARSALTDIR/$i -type d); do
 			suffixdir=${d//$PILLARSALTDIR/}
 			if [ ! -d "$local_salt_dir/$suffixdir" ]; then
-				mkdir -pv $local_salt_dir$suffixdir
+				mkdir -p $local_salt_dir$suffixdir
 			fi
 		done
 		chown -R socore:socore $local_salt_dir/$i
@@ -299,7 +310,8 @@ fail() {
 
 get_agent_count() {
 	if [ -f /opt/so/log/agents/agentstatus.log ]; then
-		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}')
+		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}' | sed 's/,//')
+		[[ -z "$AGENTCOUNT" ]] && AGENTCOUNT="0"
 	else
 		AGENTCOUNT=0
 	fi

salt/common/tools/sbin/so-common-status-check

@@ -45,7 +45,7 @@ def check_for_fps():
         result = subprocess.run([feat_full + '-mode-setup', '--is-enabled'], stdout=subprocess.PIPE)
         if result.returncode == 0:
             fps = 1
-    except FileNotFoundError:
+    except:
         fn = '/proc/sys/crypto/' + feat_full + '_enabled'
         try:
           with open(fn, 'r') as f:

salt/common/tools/sbin/so-docker-prune

@@ -4,22 +4,16 @@
 # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
-
-
-
-import sys, argparse, re, docker
+import sys, argparse, re, subprocess, json
 from packaging.version import Version, InvalidVersion
 from itertools import groupby, chain
 
-
 def get_image_name(string) -> str:
   return ':'.join(string.split(':')[:-1])
 
-
 def get_so_image_basename(string) -> str:
   return get_image_name(string).split('/so-')[-1]
 
-
 def get_image_version(string) -> str:
   ver = string.split(':')[-1]
   if ver == 'latest':
@@ -35,56 +29,75 @@ def get_image_version(string) -> str:
         return '999999.9.9'
     return ver
 
+def run_command(command):
+    process = subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    if process.returncode != 0:
+        print(f"Error executing command: {command}", file=sys.stderr)
+        print(f"Error message: {process.stderr}", file=sys.stderr)
+        exit(1)
+    return process.stdout
 
 def main(quiet):
-  client = docker.from_env()
-
-  # Prune old/stopped containers
-  if not quiet: print('Pruning old containers')
-  client.containers.prune()
-
-  image_list = client.images.list(filters={ 'dangling': False })
-
-  # Map list of image objects to flattened list of tags (format: "name:version")
-  tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list))))
-
-  # Filter to only SO images (base name begins with "so-")
-  tag_list = list(filter(lambda x: re.match(r'^.*\/so-[^\/]*$', get_image_name(x)), tag_list))
-
-  # Group tags into lists by base name (sort by same projection first)
-  tag_list.sort(key=lambda x: get_so_image_basename(x))
-  grouped_tag_lists = [ list(it) for _, it in groupby(tag_list, lambda x: get_so_image_basename(x)) ]
-
-  no_prunable = True
-  for t_list in grouped_tag_lists:
     try:
-      # Group tags by version, in case multiple images exist with the same version string
-      t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
-      grouped_t_list = [ list(it) for _,it in groupby(t_list, lambda x: get_image_version(x)) ]
-
-      # Keep the 2 most current version groups
-      if len(grouped_t_list) <= 2:
-        continue
-      else:
-        no_prunable = False
-        for group in grouped_t_list[2:]:
-          for tag in group:
-            if not quiet: print(f'Removing image {tag}')
+        # Prune old/stopped containers using docker CLI
+        if not quiet: print('Pruning old containers')
+        run_command('docker container prune -f')
+        
+        # Get list of images using docker CLI
+        images_json = run_command('docker images --format "{{json .}}"')
+        
+        # Parse the JSON output
+        image_list = []
+        for line in images_json.strip().split('\n'):
+            if line:  # Skip empty lines
+                image_list.append(json.loads(line))
+        
+        # Extract tags in the format "name:version"
+        tag_list = []
+        for img in image_list:
+            # Skip dangling images
+            if img.get('Repository') != "<none>" and img.get('Tag') != "<none>":
+                tag = f"{img.get('Repository')}:{img.get('Tag')}"
+                # Filter to only SO images (base name begins with "so-")
+                if re.match(r'^.*\/so-[^\/]*$', get_image_name(tag)):
+                    tag_list.append(tag)
+        
+        # Group tags into lists by base name (sort by same projection first)
+        tag_list.sort(key=lambda x: get_so_image_basename(x))
+        grouped_tag_lists = [list(it) for k, it in groupby(tag_list, lambda x: get_so_image_basename(x))]
+        
+        no_prunable = True
+        for t_list in grouped_tag_lists:
             try:
-              client.images.remove(tag, force=True)
-            except docker.errors.ClientError as e:
-              print(f'Could not remove image {tag}, continuing...')
-    except (docker.errors.APIError, InvalidVersion) as e:
-      print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
-      exit(1)
+                # Group tags by version, in case multiple images exist with the same version string
+                t_list.sort(key=lambda x: Version(get_image_version(x)), reverse=True)
+                grouped_t_list = [list(it) for k, it in groupby(t_list, lambda x: get_image_version(x))]
+                # Keep the 2 most current version groups
+                if len(grouped_t_list) <= 2:
+                    continue
+                else:
+                    no_prunable = False
+                    for group in grouped_t_list[2:]:
+                        for tag in group:
+                            if not quiet: print(f'Removing image {tag}')
+                            try:
+                                run_command(f'docker rmi -f {tag}')
+                            except Exception as e:
+                                print(f'Could not remove image {tag}, continuing...')
+            except (InvalidVersion) as e:
+                print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
+                exit(1)
+            except Exception as e:
+                print('Unhandled exception occurred:')
+                print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
+                exit(1)
+        
+        if no_prunable and not quiet:
+            print('No Security Onion images to prune')
+    
     except Exception as e:
-      print('Unhandled exception occurred:')
-      print(f'so-{get_so_image_basename(t_list[0])}: {e}', file=sys.stderr)
-      exit(1)
-
-  if no_prunable and not quiet:
-    print('No Security Onion images to prune')
-
+        print(f"Error: {e}", file=sys.stderr)
+        exit(1)
 
 if __name__ == "__main__":
   main_parser = argparse.ArgumentParser(add_help=False)

salt/common/tools/sbin/so-image-common

@@ -29,6 +29,7 @@ container_list() {
       "so-influxdb"
       "so-kibana"
       "so-kratos"
+      "so-hydra"
       "so-nginx"
       "so-pcaptools"
       "so-soc"
@@ -53,6 +54,7 @@ container_list() {
       "so-kafka"
       "so-kibana"
       "so-kratos"
+      "so-hydra"
       "so-logstash"
       "so-nginx"
       "so-pcaptools"

salt/common/tools/sbin/so-log-check

@@ -125,6 +125,10 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tls handshake error"          # Docker registry container when new node comes onlines
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to get license information" # Logstash trying to contact ES before it's ready
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process already finished"     # Telegraf script finished just as the auto kill timeout kicked in
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No shard available"           # Typical error when making a query before ES has finished loading all indices
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|responded with status-code 503" # telegraf getting 503 from ES during startup
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process_cluster_event_timeout_exception" # logstash waiting for elasticsearch to start
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not configured for GeoIP"     # SO does not bundle the maxminddb with Zeek
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -150,6 +154,12 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error"              # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal"  # false positive (Open Canary logging out blank IP addresses)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncing rule"                 # false positive (rule sync log line includes rule name which can contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request_unauthorized"         # false positive (login failures to Hydra result in an 'error' log) 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding index lifecycle policy" # false positive (elasticsearch policy names contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding ingest pipeline"       # false positive (elasticsearch ingest pipeline names contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating index template"      # false positive (elasticsearch index or template names contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating component template"  # false positive (elasticsearch index or template names contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|upgrading composable template" # false positive (elasticsearch composable template names contain 'error')
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -210,6 +220,8 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed"       # Detections: Exclude false positive due to automated testing
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors"                   # Detections: Not an actual error
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Initialized license manager"  # SOC log: before fields.status was changed to fields.licenseStatus
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
 fi
 
 RESULT=0
@@ -248,6 +260,9 @@ exclude_log "agentstatus.log" # ignore this log since it tracks agents in error
 exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable
 exclude_log "/nsm/kafka/data/" # ignore Kafka data directory from log check.
 
+# Include Zeek reporter.log to detect errors after running known good pcap(s) through sensor
+echo "/nsm/zeek/spool/logger/reporter.log" >> /tmp/log_check_files
+
 for log_file in $(cat /tmp/log_check_files); do
     status "Checking log file $log_file"
     tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check

salt/common/tools/sbin/so_logging_utils.py

@@ -0,0 +1,53 @@
+#!/usr/bin/python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import logging
+import os 
+import sys
+
+def setup_logging(logger_name, log_file_path, log_level=logging.INFO, format_str='%(asctime)s - %(levelname)s - %(message)s'):
+    """
+    Sets up logging for a script.
+
+    Parameters:
+        logger_name (str): The name of the logger.
+        log_file_path (str): The file path for the log file.
+        log_level (int): The logging level (e.g., logging.INFO, logging.DEBUG).
+        format_str (str): The format string for log messages.
+
+    Returns:
+        logging.Logger: Configured logger object.
+    """
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(log_level)
+
+    # Create directory for log file if it doesn't exist
+    log_file_dir = os.path.dirname(log_file_path)
+    if log_file_dir and not os.path.exists(log_file_dir):
+        try:
+            os.makedirs(log_file_dir)
+        except OSError as e:
+            print(f"Error creating directory {log_file_dir}: {e}")
+            sys.exit(1)
+
+    # Create handlers
+    c_handler = logging.StreamHandler()
+    f_handler = logging.FileHandler(log_file_path)
+    c_handler.setLevel(log_level)
+    f_handler.setLevel(log_level)
+
+    # Create formatter and add it to handlers
+    formatter = logging.Formatter(format_str)
+    c_handler.setFormatter(formatter)
+    f_handler.setFormatter(formatter)
+
+    # Add handlers to the logger if they are not already added
+    if not logger.hasHandlers():
+        logger.addHandler(c_handler)
+        logger.addHandler(f_handler)
+
+    return logger

salt/common/tools/sbin_jinja/so-import-pcap

@@ -63,7 +63,7 @@ function status {
 function pcapinfo() {
   PCAP=$1
   ARGS=$2
-  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap $ARGS
+  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS
 }
 
 function pcapfix() {
@@ -248,7 +248,7 @@ fi
 START_OLDEST_SLASH=$(echo $START_OLDEST | sed -e 's/-/%2F/g')
 END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
 if [[ $VALID_PCAPS_COUNT -gt 0 ]] || [[ $SKIPPED_PCAPS_COUNT -gt 0 ]]; then
-  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20event.module*%20%7C%20groupby%20-sankey%20event.module*%20event.dataset%20%7C%20groupby%20event.dataset%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20network.protocol%20%7C%20groupby%20rule.name%20rule.category%20event.severity_label%20%7C%20groupby%20dns.query.name%20%7C%20groupby%20file.mime_type%20%7C%20groupby%20http.virtual_host%20http.uri%20%7C%20groupby%20notice.note%20notice.message%20notice.sub_message%20%7C%20groupby%20ssl.server_name%20%7C%20groupby%20source_geo.organization_name%20source.geo.country_name%20%7C%20groupby%20destination_geo.organization_name%20destination.geo.country_name&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
+  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20event.module*%20%7C%20groupby%20-sankey%20event.module*%20event.dataset%20%7C%20groupby%20event.dataset%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20network.protocol%20%7C%20groupby%20rule.name%20rule.category%20event.severity_label%20%7C%20groupby%20dns.query.name%20%7C%20groupby%20file.mime_type%20%7C%20groupby%20http.virtual_host%20http.uri%20%7C%20groupby%20notice.note%20notice.message%20notice.sub_message%20%7C%20groupby%20ssl.server_name%20%7C%20groupby%20source.as.organization.name%20source.geo.country_name%20%7C%20groupby%20destination.as.organization.name%20destination.geo.country_name&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
 
   status "Import complete!"
   status

salt/common/tools/sbin_jinja/so-salt-emit-vm-deployment-status-event

@@ -0,0 +1,132 @@
+#!/opt/saltstack/salt/bin/python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+#
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+{%   if 'vrt' in salt['pillar.get']('features', []) -%}
+
+"""
+Script for emitting VM deployment status events to the Salt event bus.
+
+This script provides functionality to emit status events for VM deployment operations,
+used by various Security Onion VM management tools.
+
+Usage:
+    so-salt-emit-vm-deployment-status-event -v <vm_name> -H <hypervisor> -s <status>
+
+Arguments:
+    -v, --vm-name     Name of the VM (hostname_role)
+    -H, --hypervisor  Name of the hypervisor
+    -s, --status      Current deployment status of the VM
+
+Example:
+    so-salt-emit-vm-deployment-status-event -v sensor1_sensor -H hypervisor1 -s "Creating"
+"""
+
+import sys
+import argparse
+import logging
+import salt.client
+from typing import Dict, Any
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+log = logging.getLogger(__name__)
+
+def emit_event(vm_name: str, hypervisor: str, status: str) -> bool:
+    """
+    Emit a VM deployment status event to the salt event bus.
+    
+    Args:
+        vm_name: Name of the VM (hostname_role)
+        hypervisor: Name of the hypervisor
+        status: Current deployment status of the VM
+        
+    Returns:
+        bool: True if event was sent successfully, False otherwise
+    
+    Raises:
+        ValueError: If status is not a valid deployment status
+    """
+    log.info("Attempting to emit deployment event...")
+    
+    try:
+        caller = salt.client.Caller()
+        event_data = {
+            'vm_name': vm_name,
+            'hypervisor': hypervisor,
+            'status': status
+        }
+        
+        # Use consistent event tag structure
+        event_tag = f'soc/dyanno/hypervisor/{status.lower()}'
+        
+        ret = caller.cmd(
+            'event.send',
+            event_tag,
+            event_data
+        )
+        
+        if not ret:
+            log.error("Failed to emit VM deployment status event: %s", event_data)
+            return False
+            
+        log.info("Successfully emitted VM deployment status event: %s", event_data)
+        return True
+        
+    except Exception as e:
+        log.error("Error emitting VM deployment status event: %s", str(e))
+        return False
+
+def parse_args():
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description='Emit VM deployment status events to the Salt event bus.'
+    )
+    parser.add_argument('-v', '--vm-name', required=True,
+                        help='Name of the VM (hostname_role)')
+    parser.add_argument('-H', '--hypervisor', required=True,
+                        help='Name of the hypervisor')
+    parser.add_argument('-s', '--status', required=True,
+                        help='Current deployment status of the VM')
+    return parser.parse_args()
+
+def main():
+    """Main entry point for the script."""
+    try:
+        args = parse_args()
+        
+        success = emit_event(
+            vm_name=args.vm_name,
+            hypervisor=args.hypervisor,
+            status=args.status
+        )
+        
+        if not success:
+            sys.exit(1)
+            
+    except Exception as e:
+        log.error("Failed to emit status event: %s", str(e))
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()
+
+{%- else -%}
+
+echo "Hypervisor nodes are a feature supported only for customers with a valid license. \
+      Contact Security Onion Solutions, LLC via our website at https://securityonionsolutions.com \
+      for more information about purchasing a license to enable this feature."
+
+{% endif -%}

salt/docker/defaults.yaml

@@ -51,6 +51,14 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+    'so-hydra':
+      final_octet: 30
+      port_bindings:
+        - 0.0.0.0:4444:4444
+        - 0.0.0.0:4445:4445
+      custom_bind_mounts: []
+      extra_hosts: []
+      extra_env: []
     'so-logstash':
       final_octet: 29
       port_bindings:
@@ -74,6 +82,7 @@ docker:
         - 443:443
         - 8443:8443
         - 7788:7788
+        - 7789:7789
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
@@ -191,6 +200,7 @@ docker:
       final_octet: 88
       port_bindings:
         - 0.0.0.0:9092:9092
+        - 0.0.0.0:29092:29092
         - 0.0.0.0:9093:9093
         - 0.0.0.0:8778:8778
       custom_bind_mounts: []

salt/docker/soc_docker.yaml

@@ -45,6 +45,7 @@ docker:
     so-influxdb: *dockerOptions
     so-kibana: *dockerOptions
     so-kratos: *dockerOptions
+    so-hydra: *dockerOptions
     so-logstash: *dockerOptions
     so-nginx: *dockerOptions
     so-nginx-fleet-node: *dockerOptions

salt/elasticfleet/config.sls

@@ -30,6 +30,7 @@ elasticfleet_sbin:
     - user: 947
     - group: 939
     - file_mode: 755
+    - show_changes: False
 
 elasticfleet_sbin_jinja:
   file.recurse:
@@ -41,6 +42,7 @@ elasticfleet_sbin_jinja:
     - template: jinja
     - exclude_pat:
       - so-elastic-fleet-package-upgrade # exclude this because we need to watch it for changes
+    - show_changes: False
 
 eaconfdir:
   file.directory:
@@ -63,6 +65,14 @@ eastatedir:
     - group: 939
     - makedirs: True
 
+custommappingsdir:
+  file.directory:
+    - name: /nsm/custom-mappings
+    - user: 947
+    - group: 939
+    - makedirs: True
+
+
 eapackageupgrade:
   file.managed:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
@@ -73,14 +83,7 @@ eapackageupgrade:
     - template: jinja
 
 {%   if GLOBALS.role != "so-fleet" %}
-
-soresourcesrepoconfig:
-  git.config_set:
-    - name: safe.directory
-    - value: /nsm/securityonion-resources
-    - global: True
-    - user: socore
-    
+   
 {% if not GLOBALS.airgap %}
 soresourcesrepoclone:
   git.latest:
@@ -144,6 +147,7 @@ eadynamicintegration:
     - user: 947
     - group: 939
     - template: jinja
+    - show_changes: False
 
 eaintegration:
   file.recurse:
@@ -151,6 +155,7 @@ eaintegration:
     - source: salt://elasticfleet/files/integrations
     - user: 947
     - group: 939
+    - show_changes: False
 
 eaoptionalintegrationsdir:
   file.directory:
@@ -161,7 +166,7 @@ eaoptionalintegrationsdir:
 
 {% for minion in node_data %}
 {% set role = node_data[minion]["role"] %}
-{% if role in [ "eval","fleet","heavynode","import","manager","managersearch","standalone" ] %}
+{% if role in [ "eval","fleet","heavynode","import","manager", "managerhype", "managersearch","standalone" ] %}
 {% set optional_integrations = ELASTICFLEETMERGED.optional_integrations %}
 {% set integration_keys = optional_integrations.keys() %}
 fleet_server_integrations_{{ minion }}:

salt/elasticfleet/defaults.yaml

@@ -10,6 +10,8 @@ elasticfleet:
       grid_enrollment: ''
     defend_filters:
       enable_auto_configuration: False
+    subscription_integrations: False
+    auto_upgrade_integrations: False
   logging:
     zeek:
       excluded:
@@ -32,91 +34,20 @@ elasticfleet:
         - stderr
         - stdout
   packages:
-    - apache
-    - auditd
-    - auth0
-    - aws
-    - azure
-    - barracuda
-    - barracuda_cloudgen_firewall
-    - carbonblack_edr
-    - cef
-    - checkpoint
-    - cisco_asa
-    - cisco_duo
-    - cisco_ftd
-    - cisco_ios
-    - cisco_ise
-    - cisco_meraki
-    - cisco_umbrella
-    - citrix_adc
-    - citrix_waf
-    - cloudflare
-    - crowdstrike
-    - darktrace
     - elastic_agent
     - elasticsearch
     - endpoint
-    - f5_bigip
-    - fim
-    - fireeye
     - fleet_server
-    - fortinet
-    - fortinet_fortigate
-    - gcp
-    - github
-    - google_workspace
     - http_endpoint
     - httpjson
-    - iis
-    - imperva_cloud_waf
-    - journald
-    - juniper
-    - juniper_srx
-    - kafka_log
-    - lastpass
     - log
-    - m365_defender
-    - microsoft_defender_endpoint
-    - microsoft_dhcp
-    - microsoft_sqlserver
-    - mimecast
-    - mysql
-    - netflow
-    - nginx
-    - o365
-    - okta
     - osquery_manager
-    - panw
-    - pfsense
-    - proofpoint_tap
-    - pulse_connect_secure
     - redis
-    - sentinel_one
-    - snort
-    - snyk
-    - sonicwall_firewall
-    - sophos
-    - sophos_central
-    - symantec_endpoint
     - system
     - tcp
-    - tenable_io
-    - tenable_sc
-    - ti_abusech
-    - ti_anomali
-    - ti_cybersixgill
-    - ti_misp
-    - ti_otx
-    - ti_recordedfuture
-    - ti_threatq
     - udp
-    - vsphere
     - windows
     - winlog
-    - zscaler_zia
-    - zscaler_zpa
-    - 1password
   optional_integrations:
     sublime_platform:
       enabled_nodes: []

salt/elasticfleet/enabled.sls

@@ -143,12 +143,18 @@ so-elastic-fleet-integrations:
 so-elastic-agent-grid-upgrade:
   cmd.run:
     - name: /usr/sbin/so-elastic-agent-grid-upgrade
-    - retry: True
+    - retry:
+        attempts: 12
+        interval: 5
 
 so-elastic-fleet-integration-upgrade:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-integration-upgrade
 
+so-elastic-fleet-addon-integrations:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
+
 {%   if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
 so-elastic-defend-manage-filters-file-watch:
   cmd.run:

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json

@@ -0,0 +1,46 @@
+{%- set identities = salt['sqlite3.fetch']('/nsm/kratos/db/db.sqlite', 'SELECT id, json_extract(traits, "$.email") as email FROM identities;') -%}
+{%- set valid_identities = false -%}
+{%- if identities -%}
+  {%- set valid_identities = true -%}
+  {%- for id, email in identities -%}
+    {%- if not id or not email -%}
+      {%- set valid_identities = false -%}
+      {%- break -%}
+    {%- endif -%}
+  {%- endfor -%}
+{%- endif -%}
+{
+  "package": {
+    "name": "log",
+    "version": ""
+  },
+  "name": "kratos-logs",
+  "namespace": "so",
+  "description": "Kratos logs",
+  "policy_id": "so-grid-nodes_general",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.logs": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/kratos/kratos.log"
+            ],
+            "data_stream.dataset": "kratos",
+            "tags": ["so-kratos"],
+            {%- if valid_identities -%}
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos\n- if:\n    has_fields:\n      - identity_id\n  then:{% for id, email in identities %}\n    - if:\n        equals:\n          identity_id: \"{{ id }}\"\n      then:\n        - add_fields:\n            target: ''\n            fields:\n              user.name: \"{{ email }}\"{% endfor %}",
+            {%- else -%}
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos",
+            {%- endif -%}
+            "custom": "pipeline: kratos"
+          }
+        }
+      }
+    }
+  },
+  "force": true
+}
+

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -1,27 +1,33 @@
 {
-	"name": "elastic-defend-endpoints",
-	"namespace": "default",
-	"description": "",
-	"package": {
-		"name": "endpoint",
-		"title": "Elastic Defend",
-		"version": "8.14.0"
-	},
-	"enabled": true,
-	"policy_id": "endpoints-initial",
-	"inputs": [{
-		"type": "ENDPOINT_INTEGRATION_CONFIG",
-		"enabled": true,
-		"streams": [],
-		"config": {
-			"_config": {
-				"value": {
-					"type": "endpoint",
-					"endpointConfig": {
-						"preset": "DataCollection"
-					}
-				}
-			}
-		}
-	}]
-}
+  "name": "elastic-defend-endpoints",
+  "namespace": "default",
+  "description": "",
+  "package": {
+    "name": "endpoint",
+    "title": "Elastic Defend",
+    "version": "8.18.1",
+    "requires_root": true
+  },
+  "enabled": true,
+  "policy_ids": [
+    "endpoints-initial"
+  ],
+  "vars": {},
+  "inputs": [
+    {
+      "type": "ENDPOINT_INTEGRATION_CONFIG",
+      "enabled": true,
+      "config": {
+        "_config": {
+          "value": {
+            "type": "endpoint",
+            "endpointConfig": {
+              "preset": "DataCollection"
+            }
+          }
+        }
+      },
+      "streams": []
+    }
+  ]
+}

salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json

@@ -3,9 +3,9 @@
     "name": "log",
     "version": ""
   },
-  "name": "kratos-logs",
+  "name": "hydra-logs",
   "namespace": "so",
-  "description": "Kratos logs",
+  "description": "Hydra logs",
   "policy_id": "so-grid-nodes_general",
   "inputs": {
     "logs-logfile": {
@@ -15,12 +15,12 @@
           "enabled": true,
           "vars": {
             "paths": [
-              "/opt/so/log/kratos/kratos.log"
+              "/opt/so/log/hydra/hydra.log"
             ],
-            "data_stream.dataset": "kratos",
-            "tags": ["so-kratos"],
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos",
-            "custom": "pipeline: kratos"
+            "data_stream.dataset": "hydra",
+            "tags": ["so-hydra"],
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: hydra",
+            "custom": "pipeline: hydra"
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/idh-logs.json

@@ -19,7 +19,7 @@
             ],
             "data_stream.dataset": "idh",
             "tags": [],
-            "processors": "\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- drop_fields:\n    when:\n      equals:\n        logtype: \"1001\"\n    fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n    ignore_missing: true\n- rename:\n    fields:\n      - from: \"src_host\"\n        to: \"source.ip\"\n      - from: \"src_port\"\n        to: \"source.port\"\n      - from: \"dst_host\"\n        to: \"destination.host\"\n      - from: \"dst_port\"\n        to: \"destination.port\"\n    ignore_missing: true\n- convert:\n    fields:\n      - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n    ignore_missing: true\n- drop_fields:\n    fields: '[\"prospector\", \"input\", \"offset\", \"beat\"]'\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: opencanary",
+            "processors": "\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- convert:\n    fields:\n      - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n- drop_fields:\n    when:\n      equals:\n        event.code: \"1001\"\n    fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n    ignore_missing: true\n- rename:\n    fields:\n      - from: \"src_host\"\n        to: \"source.ip\"\n      - from: \"src_port\"\n        to: \"source.port\"\n      - from: \"dst_host\"\n        to: \"destination.host\"\n      - from: \"dst_port\"\n        to: \"destination.port\"\n    ignore_missing: true\n- drop_fields:\n    fields: '[\"prospector\", \"input\", \"offset\", \"beat\"]'\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: opencanary",
             "custom": "pipeline: common"
           }
         }

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -20,7 +20,7 @@
             ],
             "data_stream.dataset": "import",
             "custom": "",
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-1.59.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-1.45.1\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-1.59.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-1.59.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-1.45.1\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-2.3.3\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-3.1.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-2.3.3\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-2.3.3\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-3.1.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ]

salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json

@@ -0,0 +1,35 @@
+{
+  "package": {
+    "name": "log",
+    "version": ""
+  },
+  "name": "so-ip-mappings",
+  "namespace": "so",
+  "description": "IP Description mappings",
+  "policy_id": "so-grid-nodes_general",
+  "vars": {},
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.logs": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/nsm/custom-mappings/ip-descriptions.csv"
+            ],
+            "data_stream.dataset": "hostnamemappings",
+            "tags": [
+              "so-ip-mappings"
+            ],
+            "processors": "- decode_csv_fields:\n    fields:\n      message: decoded.csv\n    separator: \",\"\n    ignore_missing: false\n    overwrite_keys: true\n    trim_leading_space: true\n    fail_on_error: true\n\n- extract_array:\n    field: decoded.csv\n    mappings:\n      so.ip_address: '0'\n      so.description: '1'\n\n- script:\n    lang: javascript\n    source: >\n      function process(event) {\n          var ip = event.Get('so.ip_address');\n          var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n          if (!validIpRegex.test(ip)) {\n              event.Cancel();\n          }\n      }\n- fingerprint:\n    fields: [\"so.ip_address\"]\n    target_field: \"@metadata._id\"\n",
+            "custom": ""
+          }
+        }
+      }
+    }
+  },
+  "force": true
+}
+
+

salt/elasticfleet/files/integrations/grid-nodes_general/syslog-tcp-514.json

@@ -11,7 +11,7 @@
     "tcp-tcp": {
       "enabled": true,
       "streams": {
-        "tcp.generic": {
+        "tcp.tcp": {
           "enabled": true,
           "vars": {
             "listen_address": "0.0.0.0",
@@ -23,7 +23,8 @@
               "syslog"
             ],
             "syslog_options": "field: message\n#format: auto\n#timezone: Local",
-            "ssl": ""
+            "ssl": "",
+            "custom": ""
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json

@@ -11,7 +11,7 @@
     "udp-udp": {
       "enabled": true,
       "streams": {
-        "udp.generic": {
+        "udp.udp": {
           "enabled": true,
           "vars": {
             "listen_address": "0.0.0.0",
@@ -20,11 +20,13 @@
             "pipeline": "syslog",
             "max_message_size": "10KiB",
             "keep_null": false,
-            "processors": "- add_fields:\n    target: event\n    fields: \n      module: syslog\n",
+            "processors": "- add_fields:\n    target: event\n    fields: \n      module: syslog",
             "tags": [
               "syslog"
             ],
-            "syslog_options": "field: message\n#format: auto\n#timezone: Local"
+            "syslog_options": "field: message\n#format: auto\n#timezone: Local\n",
+            "preserve_original_event": false,
+            "custom": ""
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json

@@ -31,7 +31,8 @@
             ],
             "tags": [
               "so-grid-node"
-            ]
+            ],
+            "processors": "- if:\n    contains:\n      message: \"salt-minion\"\n  then: \n    - dissect:\n        tokenizer: \"%{} %{} %{} %{} %{} %{}: [%{log.level}] %{*}\"\n        field: \"message\"\n        trim_values: \"all\"\n        target_prefix: \"\"\n    - drop_event:\n        when:\n          equals:\n            log.level: \"INFO\""
           }
         }
       }

salt/elasticfleet/integration-defaults.map.jinja

@@ -0,0 +1,158 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+  this file except in compliance with the Elastic License 2.0. #}
+
+
+{% import_json '/opt/so/state/esfleet_package_components.json' as ADDON_PACKAGE_COMPONENTS %}
+{% import_json '/opt/so/state/esfleet_component_templates.json' as INSTALLED_COMPONENT_TEMPLATES %}
+{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+
+{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
+{% set ADDON_INTEGRATION_DEFAULTS = {} %}
+
+{# Some fleet integrations don't follow the standard naming convention #}
+{% set WEIRD_INTEGRATIONS = {
+    'awsfirehose.logs': 'awsfirehose',
+    'awsfirehose.metrics': 'aws.cloudwatch',
+    'cribl.logs': 'cribl',
+    'cribl.metrics': 'cribl',
+    'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
+    'azure_application_insights.app_insights': 'azure.app_insights',
+    'azure_application_insights.app_state': 'azure.app_state',
+    'azure_billing.billing': 'azure.billing',
+    'azure_functions.metrics': 'azure.function',
+    'azure_metrics.compute_vm_scaleset': 'azure.compute_vm_scaleset',
+    'azure_metrics.compute_vm': 'azure.compute_vm',
+    'azure_metrics.container_instance': 'azure.container_instance',
+    'azure_metrics.container_registry': 'azure.container_registry',
+    'azure_metrics.container_service': 'azure.container_service',
+    'azure_metrics.database_account': 'azure.database_account',
+    'azure_metrics.monitor': 'azure.monitor',
+    'azure_metrics.storage_account': 'azure.storage_account',
+    'azure_openai.metrics': 'azure.open_ai',
+    'beat.state': 'beats.stack_monitoring.state',
+    'beat.stats': 'beats.stack_monitoring.stats',
+    'enterprisesearch.health': 'enterprisesearch.stack_monitoring.health',
+    'enterprisesearch.stats': 'enterprisesearch.stack_monitoring.stats',
+    'kibana.cluster_actions': 'kibana.stack_monitoring.cluster_actions',
+    'kibana.cluster_rules': 'kibana.stack_monitoring.cluster_rules',
+    'kibana.node_actions': 'kibana.stack_monitoring.node_actions',
+    'kibana.node_rules': 'kibana.stack_monitoring.node_rules',
+    'kibana.stats': 'kibana.stack_monitoring.stats',
+    'kibana.status': 'kibana.stack_monitoring.status',
+    'logstash.node_cel': 'logstash.stack_monitoring.node',
+    'logstash.node_stats': 'logstash.stack_monitoring.node_stats',
+    'synthetics.browser': 'synthetics-browser',
+    'synthetics.browser_network': 'synthetics-browser.network',
+    'synthetics.browser_screenshot': 'synthetics-browser.screenshot',
+    'synthetics.http': 'synthetics-http',
+    'synthetics.icmp': 'synthetics-icmp',
+    'synthetics.tcp': 'synthetics-tcp',
+    'swimlane.swimlane_api': 'swimlane.api',
+    'swimlane.tenant_api': 'swimlane.tenant',
+    'swimlane.turbine_api': 'turbine.api'
+   } %}
+
+{% for pkg in ADDON_PACKAGE_COMPONENTS %}
+{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
+{#     skip core integrations #}
+{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
+{#     generate defaults for each integration #}
+{%     if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
+{%       for pattern in pkg.es_index_patterns %}
+{%         if "metrics-" in pattern.name %}
+{%           set integration_type = "metrics-" %}
+{%         elif "logs-" in pattern.name %}
+{%           set integration_type = "logs-" %}
+{%         else %}
+{%           set integration_type = "" %}
+{%         endif %}
+{%         set component_name = pkg.name ~ "." ~ pattern.title %}
+{%         set index_pattern = pattern.name %}
+
+{#         fix weirdly named components #}
+{%         if component_name in WEIRD_INTEGRATIONS %}
+{%           set component_name = WEIRD_INTEGRATIONS[component_name] %}
+{%         endif %}
+
+{#         create duplicate of component_name, so we can split generics from @custom component templates in the index template below and overwrite the default @package when needed
+           eg. having to replace unifiedlogs.generic@package with filestream.generic@package, but keep the ability to customize unifiedlogs.generic@custom and its ILM policy #}
+{%         set custom_component_name = component_name %}
+
+{#         duplicate integration_type to assist with sometimes needing to overwrite component templates with 'logs-filestream.generic@package' (there is no metrics-filestream.generic@package) #}
+{%         set generic_integration_type = integration_type %}
+
+{#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
+{%           set component_name_x = component_name.replace(".","_x_") %}
+{#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
+{%           set integration_key = "so-" ~ integration_type ~ component_name_x %}
+
+{#         if its a .generic template make sure that a .generic@package for the integration exists. Else default to logs-filestream.generic@package #}
+{%         if ".generic" in component_name and integration_type ~ component_name ~ "@package" not in INSTALLED_COMPONENT_TEMPLATES %}
+{#           these generic templates by default are directed to index_pattern of 'logs-generic-*', overwrite that here to point to eg gcp_pubsub.generic-* #}
+{%           set index_pattern = integration_type ~ component_name ~ "-*" %}
+{#           includes use of .generic component template, but it doesn't exist in installed component templates. Redirect it to filestream.generic@package #}
+{%           set component_name = "filestream.generic" %}
+{%           set generic_integration_type = "logs-" %}
+{%         endif %}
+
+{#           Default integration settings #}
+{%           set integration_defaults = {
+               "index_sorting": false,
+               "index_template": {
+                 "composed_of": [generic_integration_type ~ component_name ~ "@package", integration_type ~ custom_component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                 "data_stream": {
+                   "allow_custom_routing": false,
+                   "hidden": false
+                 },
+                 "ignore_missing_component_templates": [integration_type ~ custom_component_name ~ "@custom"],
+                 "index_patterns": [index_pattern],
+                 "priority": 501,
+                 "template": {
+                   "settings": {
+                     "index": {
+                       "lifecycle": {"name": "so-" ~ integration_type ~ custom_component_name ~ "-logs"},
+                         "number_of_replicas": 0
+                     }
+                   }
+                 }
+               },
+               "policy": {
+                 "phases": {
+                   "cold": {
+                     "actions": {
+                       "set_priority": {"priority": 0}
+                     },
+                     "min_age": "60d"
+                   },
+                   "delete": {
+                     "actions": {
+                       "delete": {}
+                     },
+                     "min_age": "365d"
+                   },
+                   "hot": {
+                     "actions": {
+                       "rollover": {
+                         "max_age": "30d",
+                         "max_primary_shard_size": "50gb"
+                       },
+                       "set_priority": {"priority": 100}
+                      },
+                      "min_age": "0ms"
+                   },
+                   "warm": {
+                     "actions": {
+                       "set_priority": {"priority": 50}
+                     },
+                     "min_age": "30d"
+                   }
+                 }
+               }
+             } %}
+
+{%         do ADDON_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
+{%       endfor %}
+{%     endif %}
+{%   endif %}
+{% endfor %}

salt/elasticfleet/soc_elasticfleet.yaml

@@ -40,6 +40,16 @@ elasticfleet:
         global: True
         helpLink: elastic-fleet.html
         advanced: True
+    subscription_integrations:
+      description: Enable the installation of integrations that require an Elastic license.
+      global: True
+      forcedType: bool
+      helpLink: elastic-fleet.html
+    auto_upgrade_integrations:
+      description: Enables or disables automatically upgrading Elastic Agent integrations.
+      global: True
+      forcedType: bool
+      helpLink: elastic-fleet.html
     server:
       custom_fqdn:
         description: Custom FQDN for Agents to connect to. One per line.

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -88,7 +88,13 @@ elastic_fleet_package_version_check() {
 
 elastic_fleet_package_latest_version_check() {
     PACKAGE=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.latestVersion'
+    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages/$PACKAGE" --fail); then
+        if version=$(jq -e -r '.item.latestVersion' <<< $output); then
+            echo "$version"
+        fi
+    else
+        echo "Error: Failed to get latest version for $PACKAGE"
+    fi
 }
 
 elastic_fleet_package_install() {
@@ -97,11 +103,20 @@ elastic_fleet_package_install() {
     curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  -d '{"force":true}' "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION"
 }
 
+elastic_fleet_bulk_package_install() {
+    BULK_PKG_LIST=$1
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$1 "localhost:5601/api/fleet/epm/packages/_bulk"
+}
+
 elastic_fleet_package_is_installed() {
     PACKAGE=$1
     curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.status'
 }
 
+elastic_fleet_installed_packages() {
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=500"
+}
+
 elastic_fleet_agent_policy_ids() {
     curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].id
     if [ $? -ne 0 ]; then
@@ -140,9 +155,13 @@ elastic_fleet_integration_policy_package_name() {
 elastic_fleet_integration_policy_package_version() {
     AGENT_POLICY=$1
     INTEGRATION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version'
-    if [ $? -ne 0 ]; then
-        echo "Error: Failed to retrieve package version for '$INTEGRATION' in '$AGENT_POLICY'."
+
+    if output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" --fail); then
+        if version=$(jq -e -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version' <<< $output); then
+            echo "$version"
+        fi
+    else
+        echo "Error: Failed to retrieve agent policy $AGENT_POLICY"
         exit 1
     fi
 }

salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade

@@ -1,62 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-elastic-fleet-common
-
-curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/)
-if [ $? -ne 0 ]; then
-    echo "Error: Failed to connect to Kibana."
-    exit 1
-fi
-
-IFS=$'\n'
-agent_policies=$(elastic_fleet_agent_policy_ids)
-if [ $? -ne 0 ]; then
-    echo "Error: Failed to retrieve agent policies."
-    exit 1
-fi
-
-for AGENT_POLICY in $agent_policies; do
-    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
-    for INTEGRATION in $integrations; do
-        if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
-            # Get package name so we know what package to look for when checking the current and latest available version
-            PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
-
-            # Get currently installed version of package
-            PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION")
-
-            # Get latest available version of package
-            AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME")
-
-            # Get integration ID
-            INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION")
-
-            if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
-                # Dry run of the upgrade
-                echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
-                echo "Upgrading $INTEGRATION..."
-                echo "Starting dry run..."
-                DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID")
-                DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
-                
-                # If no errors with dry run, proceed with actual upgrade
-                if [[ "$DRYRUN_ERRORS" == "false" ]]; then
-                    echo "No errors detected. Proceeding with upgrade..."
-                    elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
-                    if [ $? -ne 0 ]; then
-                        echo "Error: Upgrade failed for integration ID '$INTEGRATION_ID'."
-                        exit 1
-                    fi
-                else
-                    echo "Errors detected during dry run. Stopping upgrade..."
-                    exit 1
-                fi
-            fi
-        fi
-    done
-done
-echo

salt/elasticfleet/tools/sbin/so-elastic-fleet-package-list

@@ -10,6 +10,6 @@
 SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
 
 # List configured package policies
-curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages" -H 'kbn-xsrf: true' | jq
+curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages?prerelease=true" -H 'kbn-xsrf: true' | jq
 
 echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -76,5 +76,17 @@ do
     printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done
 
-printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n"
+printf "\n\n### Generating MSI...\n"
+cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
+docker run \
+--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \
+{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
+printf "\n### MSI Generated...\n"
+
+printf "\n### Cleaning up temp files \n"
 rm -rf /nsm/elastic-agent-workspace
+rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
+
+printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n"
+\cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/
+chmod 644 /nsm/elastic-fleet/so_agent-installers/*

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade

@@ -14,7 +14,7 @@ if ! is_manager_node; then
 fi
 
 # Get current list of Grid Node Agents that need to be upgraded
-RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=policy_id%20%3A%20so-grid-nodes_%2A&showInactive=false&showUpgradeable=true&getStatusSummary=true")
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%20:%20%22{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%22%20and%20policy_id%20:%20%22so-grid-nodes_general%22&showInactive=false&getStatusSummary=true")
 
 # Check to make sure that the server responded with good data - else, bail from script
 CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON")

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade

@@ -0,0 +1,81 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
+{%- set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %}
+
+. /usr/sbin/so-elastic-fleet-common
+
+curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/)
+if [ $? -ne 0 ]; then
+    echo "Error: Failed to connect to Kibana."
+    exit 1
+fi
+
+IFS=$'\n'
+agent_policies=$(elastic_fleet_agent_policy_ids)
+if [ $? -ne 0 ]; then
+    echo "Error: Failed to retrieve agent policies."
+    exit 1
+fi
+
+default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
+
+for AGENT_POLICY in $agent_policies; do
+    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+    for INTEGRATION in $integrations; do
+        if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
+            # Get package name so we know what package to look for when checking the current and latest available version
+            PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
+            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+            if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
+            {%- endif %}
+                # Get currently installed version of package
+                attempt=0
+                max_attempts=3
+                while [ $attempt -lt $max_attempts ]; do
+                    if PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION") && AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME"); then
+                        break
+                    fi
+                    attempt=$((attempt + 1))
+                done
+                if [ $attempt -eq $max_attempts ]; then
+                    echo "Error: Failed getting $PACKAGE_VERSION or $AVAILABLE_VERSION"
+                    exit 1
+                fi
+
+                # Get integration ID
+                INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION")
+
+                if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
+                    # Dry run of the upgrade
+                    echo ""
+                    echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
+                    echo "Upgrading $INTEGRATION..."
+                    echo "Starting dry run..."
+                    DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID")
+                    DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
+
+                    # If no errors with dry run, proceed with actual upgrade
+                    if [[ "$DRYRUN_ERRORS" == "false" ]]; then
+                        echo "No errors detected. Proceeding with upgrade..."
+                        elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
+                        if [ $? -ne 0 ]; then
+                            echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'."
+                            exit 1
+                        fi
+                    else
+                        echo "Errors detected during dry run for $PACKAGE_NAME policy upgrade..."
+                        exit 1
+                    fi
+                fi
+            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+            fi
+            {%- endif %}
+        fi
+    done
+done
+echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -0,0 +1,189 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+# this file except in compliance with the Elastic License 2.0.
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{% set SUB = salt['pillar.get']('elasticfleet:config:subscription_integrations', default=false) %}
+{% set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %}
+{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
+
+. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
+
+# Check that /opt/so/state/estemplates.txt exists to signal that Elasticsearch
+#  has completed its first run of core-only integrations/indices/components/ilm
+STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
+INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
+BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
+BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
+BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
+PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
+COMPONENT_TEMPLATES=/opt/so/state/esfleet_component_templates.json
+
+PENDING_UPDATE=false
+
+# Integrations which are included in the package registry, but excluded from automatic installation via this script.
+#   Requiring some level of manual Elastic Stack configuration before installation
+EXCLUDED_INTEGRATIONS=('apm')
+
+version_conversion(){
+    version=$1
+    echo "$version" | awk -F '.' '{ printf("%d%03d%03d\n", $1, $2, $3); }'
+}
+
+compare_versions() {
+    version1=$1
+    version2=$2
+
+    # Convert versions to numbers
+    num1=$(version_conversion "$version1")
+    num2=$(version_conversion "$version2")
+
+    # Compare using bc
+    if (( $(echo "$num1 < $num2" | bc -l) )); then
+        echo "less"
+    elif (( $(echo "$num1 > $num2" | bc -l) )); then
+        echo "greater"
+    else
+        echo "equal"
+    fi
+}
+
+IFS=$'\n'
+agent_policies=$(elastic_fleet_agent_policy_ids)
+if [ $? -ne 0 ]; then
+    echo "Error: Failed to retrieve agent policies."
+    exit 1
+fi
+
+default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
+
+in_use_integrations=()
+
+for AGENT_POLICY in $agent_policies; do
+    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+    for INTEGRATION in $integrations; do
+        PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
+        # non-default integrations that are in-use in any policy
+        if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
+            in_use_integrations+=("$PACKAGE_NAME")
+        fi
+    done
+done
+
+if [[ -f $STATE_FILE_SUCCESS  ]]; then
+    if retry 3 1 "curl -s -K /opt/so/conf/elasticsearch/curl.config --output /dev/null --silent --head --fail localhost:5601/api/fleet/epm/packages"; then
+        # Package_list contains all integrations beta / non-beta.
+        latest_package_list=$(/usr/sbin/so-elastic-fleet-package-list)
+        echo '{ "packages" : []}' > $BULK_INSTALL_PACKAGE_LIST
+        rm -f $INSTALLED_PACKAGE_LIST
+        echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .savedObject.attributes.install_version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
+
+        while read -r package; do
+            # get package details
+            package_name=$(echo "$package" | jq -r '.name')
+            latest_version=$(echo "$package" | jq -r '.latest_version')
+            installed_version=$(echo "$package" | jq -r '.installed_version')
+            subscription=$(echo "$package" | jq -r '.subscription')
+            bulk_package=$(echo "$package" | jq '{name: .name, version: .latest_version}' )
+
+            if [[ ! "${EXCLUDED_INTEGRATIONS[@]}" =~ "$package_name" ]]; then
+            {% if not SUB %}
+                if [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then
+                    # pass over integrations that require non-basic elastic license
+                    echo "$package_name integration requires an Elastic license of $subscription or greater... skipping"
+                    continue
+                else
+                    if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then
+                        echo "$package_name is not installed... Adding to next update."
+                        jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+
+                        PENDING_UPDATE=true
+                    else
+                        results=$(compare_versions "$latest_version" "$installed_version")
+                        if [ $results == "greater" ]; then
+                            {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations  #}
+                            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                            if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
+                            {%- endif %}
+                                echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
+                                jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+
+                                PENDING_UPDATE=true
+                            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                            else
+                                echo "skipping available upgrade for in use integration - $package_name."
+                            fi
+                            {%- endif %}
+                        fi
+                    fi
+                fi
+            {% else %}
+                if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then
+                    echo "$package_name is not installed... Adding to next update."
+                    jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                    PENDING_UPDATE=true
+                else
+                    results=$(compare_versions "$latest_version" "$installed_version")
+                    if [ $results == "greater" ]; then
+                        {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations  #}
+                        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                        if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
+                        {%- endif %}
+                            echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
+                            jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                            PENDING_UPDATE=true
+                        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                        else
+                            echo "skipping available upgrade for in use integration - $package_name."
+                        fi
+                        {%- endif %}
+                    fi
+                fi
+            {% endif %}
+            else
+                echo "Skipping $package_name..."
+            fi
+        done <<< "$(jq -c '.packages[]' "$INSTALLED_PACKAGE_LIST")"
+
+        if [ "$PENDING_UPDATE" = true ]; then
+            # Run chunked install of packages
+            echo "" > $BULK_INSTALL_OUTPUT
+            pkg_group=1
+            pkg_filename="${BULK_INSTALL_PACKAGE_LIST%.json}"
+
+            jq -c '.packages | _nwise(25)' $BULK_INSTALL_PACKAGE_LIST | while read -r line; do
+                echo "$line" | jq '{ "packages": . }' > "${pkg_filename}_${pkg_group}.json"
+                pkg_group=$((pkg_group + 1))
+            done
+
+            for file in "${pkg_filename}_"*.json; do
+                [ -e "$file" ] || continue
+                elastic_fleet_bulk_package_install $file >> $BULK_INSTALL_OUTPUT
+            done
+            # cleanup any temp files for chunked package install
+            rm -f ${pkg_filename}_*.json $BULK_INSTALL_PACKAGE_LIST
+        else
+            echo "Elastic integrations don't appear to need installation/updating..."
+        fi
+        # Write out file for generating index/component/ilm templates
+        latest_installed_package_list=$(elastic_fleet_installed_packages)
+        echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+        if retry 3 1 "so-elasticsearch-query / --fail --output /dev/null"; then
+            # Refresh installed component template list
+            latest_component_templates_list=$(so-elasticsearch-query _component_template | jq '.component_templates[] | .name' | jq -s '.')
+            echo $latest_component_templates_list > $COMPONENT_TEMPLATES
+        fi
+
+    else
+        # This is the installation of add-on integrations and upgrade of existing integrations. Exiting without error, next highstate will attempt to re-run.
+        echo "Elastic Fleet does not appear to be responding... Exiting... "
+        exit 0
+    fi
+else
+    # This message will appear when an update to core integration is made and this script is run at the same time as
+    #  elasticsearch.enabled -> detects change to core index settings -> deletes estemplates.txt
+    echo "Elasticsearch may not be fully configured yet or is currently updating core index settings."
+    exit 0
+fi

salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy

@@ -32,7 +32,7 @@ if ! echo "$output" | grep -q "so-manager_kafka"; then
                 --arg KAFKACA "$KAFKACA" \
                 --arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \
                 --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
-                    '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-securityonion","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
+                    '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 10 }, "topics":[{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
                 )
   curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" -o /dev/null
   refresh_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)

salt/elasticsearch/auth.sls

@@ -15,7 +15,7 @@
 elastic_auth_pillar:
   file.managed:
     - name: /opt/so/saltstack/local/pillar/elasticsearch/auth.sls
-    - mode: 600
+    - mode: 640
     - reload_pillar: True
     - contents: |
         elasticsearch:

salt/elasticsearch/config.map.jinja

@@ -28,7 +28,7 @@
 {%   endfor %}
 {% endfor %}
 
-{% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %}
+{% if grains.id.split('_') | last in ['manager','managerhype','managersearch','standalone'] %}
     {% if ELASTICSEARCH_SEED_HOSTS | length > 1 %}
       {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %}
       {% for NODE in ELASTICSEARCH_SEED_HOSTS %}

salt/elasticsearch/config.sls

@@ -47,6 +47,7 @@ elasticsearch_sbin:
     - file_mode: 755
     - exclude_pat:
       - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
+    - show_changes: False
 
 elasticsearch_sbin_jinja:
   file.recurse:
@@ -60,6 +61,7 @@ elasticsearch_sbin_jinja:
       - so-elasticsearch-ilm-policy-load # exclude this because we need to watch it for changes, we sync it in another state
     - defaults:
         GLOBALS: {{ GLOBALS }}
+    - show_changes: False
 
 so-elasticsearch-ilm-policy-load-script:
   file.managed:
@@ -69,6 +71,7 @@ so-elasticsearch-ilm-policy-load-script:
     - group: 939
     - mode: 754
     - template: jinja
+    - show_changes: False
 
 so-elasticsearch-pipelines-script:
   file.managed:
@@ -77,6 +80,7 @@ so-elasticsearch-pipelines-script:
     - user: 930
     - group: 939
     - mode: 754
+    - show_changes: False
 
 esingestdir:
   file.directory:
@@ -110,6 +114,7 @@ esingestdynamicconf:
     - user: 930
     - group: 939
     - template: jinja
+    - show_changes: False
 
 esingestconf:
   file.recurse:
@@ -117,6 +122,7 @@ esingestconf:
     - source: salt://elasticsearch/files/ingest
     - user: 930
     - group: 939
+    - show_changes: False
 
 # Remove .fleet_final_pipeline-1 because we are using global@custom now
 so-fleet-final-pipeline-remove:
@@ -153,6 +159,7 @@ esyml:
     - defaults:
         ESCONFIG: {{ ELASTICSEARCHMERGED.config }}
     - template: jinja
+    - show_changes: False
 
 esroles:
   file.recurse:
@@ -162,6 +169,7 @@ esroles:
     - template: jinja
     - user: 930
     - group: 939
+    - show_changes: False
 
 nsmesdir:
   file.directory:

salt/elasticsearch/enabled.sls

@@ -38,7 +38,7 @@ so-elasticsearch:
       {% endfor %}
     {% endif %}
     - environment:
-      {% if ELASTICSEARCH_SEED_HOSTS | length == 1 or GLOBALS.role == 'so-heavynode' %}
+      {% if (GLOBALS.role in GLOBALS.manager_roles and ELASTICSEARCH_SEED_HOSTS | length == 1) or GLOBALS.role == 'so-heavynode' %}
       - discovery.type=single-node
       {% endif %}
       - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true
@@ -116,6 +116,7 @@ escomponenttemplates:
     - clean: True
     - onchanges_in:
       - file: so-elasticsearch-templates-reload
+    - show_changes: False
       
 # Auto-generate templates from defaults file
 {%     for index, settings in ES_INDEX_SETTINGS.items() %}
@@ -127,6 +128,7 @@ es_index_template_{{index}}:
     - defaults:
       TEMPLATE_CONFIG: {{ settings.index_template }}
     - template: jinja
+    - show_changes: False
     - onchanges_in:
       - file: so-elasticsearch-templates-reload
 {%       endif %}
@@ -146,12 +148,13 @@ es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
 {%         endif %}
     - user: 930
     - group: 939
+    - show_changes: False
     - onchanges_in:
       - file: so-elasticsearch-templates-reload
 {%       endfor %}
 {%     endif %}
 
-{% if GLOBALS.role in GLOBALS.manager_roles %}
+{%     if GLOBALS.role in GLOBALS.manager_roles %}
 so-es-cluster-settings:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-cluster-settings
@@ -160,7 +163,7 @@ so-es-cluster-settings:
     - require:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
-{% endif %}
+{%     endif %}
 
 so-elasticsearch-ilm-policy-load:
   cmd.run:
@@ -201,12 +204,17 @@ so-elasticsearch-roles-load:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
 
-{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+{%     if grains.role in ['so-managersearch', 'so-manager', 'so-managerhype'] %}
+{%       set ap = "absent" %}
+{%     endif %}
+{%     if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
 {%       if ELASTICSEARCHMERGED.index_clean %}
 {%         set ap = "present" %}
 {%       else %}
 {%         set ap = "absent" %}
 {%       endif %}
+{%     endif %}  
+{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
 so-elasticsearch-indices-delete:
   cron.{{ap}}:
     - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1

salt/elasticsearch/files/ingest-dynamic/common

@@ -26,7 +26,7 @@
     {
         "geoip":  {
                 "field": "destination.ip",
-                "target_field": "destination_geo",
+                "target_field": "destination.as",
                 "database_file": "GeoLite2-ASN.mmdb",
                 "ignore_missing": true,
                 "ignore_failure": true,
@@ -36,13 +36,17 @@
     {
         "geoip":  {
                 "field": "source.ip",
-                "target_field": "source_geo",
+                "target_field": "source.as",
                 "database_file": "GeoLite2-ASN.mmdb",
                 "ignore_missing": true,
                 "ignore_failure": true,
                 "properties": ["ip", "asn", "organization_name", "network"]
         }
     },
+    { "rename":          { "field": "destination.as.organization_name", "target_field": "destination.as.organization.name", "ignore_failure": true, "ignore_missing": true  } },
+    { "rename":          { "field": "source.as.organization_name", "target_field": "source.as.organization.name", "ignore_failure": true, "ignore_missing": true  } },
+    { "rename":          { "field": "destination.as.asn", "target_field": "destination.as.number", "ignore_failure": true, "ignore_missing": true  } },
+    { "rename":          { "field": "source.as.asn", "target_field": "source.as.number", "ignore_failure": true, "ignore_missing": true  } },
     { "set":             { "if": "ctx.event?.severity == 1",   "field": "event.severity_label", "value": "low", "override": true }  },
     { "set":             { "if": "ctx.event?.severity == 2",   "field": "event.severity_label", "value": "medium", "override": true }  },
     { "set":             { "if": "ctx.event?.severity == 3",   "field": "event.severity_label", "value": "high", "override": true }  },

salt/elasticsearch/files/ingest/global@custom

@@ -8,20 +8,23 @@
   "processors": [
     { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
     { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
+    { "split":      { "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')", "field":"data_stream.dataset", "separator":"\\.", "target_field":"datastream_dataset_temp", "ignore_missing":true } },
     { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
+    { "set":        { "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'", "field":"event.module", "value":"{{ datastream_dataset_temp.0 }}", "ignore_failure":true, "ignore_empty_value":true, "description":"Fix EA network packet capture" } },
     { "gsub":       { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
-    { "append":     { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}"  } },
+    { "append":     { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}", "allow_duplicates": false  } },
     { "set":        { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
     { "set":        { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
     { "set":        { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
     { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
-    { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } },
-    { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } },
-    { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
+    { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.dataset", "value": "import" } },
+    { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.namespace", "value": "so" } },
     { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
     { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
-    { "rename":        { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
+    { "rename":       { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
     { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
-    { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } 
+    {"append":      {"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"if":"ctx?.event?.dataset == 'endpoint.events.network' && ctx?.source?.ip != null","ignore_failure":true}},
+    {"foreach":     {"field":"host.ip","processor":{"append":{"field":"related.ip","value":"{{_ingest._value}}","allow_duplicates":false}},"if":"ctx?.event?.module == 'endpoint'","description":"Extract IPs from Elastic Agent events (host.ip) and adds them to related.ip"}},
+    { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
   ]
 }

salt/elasticsearch/files/ingest/hydra

@@ -0,0 +1,9 @@
+{
+  "description" : "hydra",
+  "processors" : [
+    {"set":{"field":"audience","value":"access","override":false,"ignore_failure":true}},
+    {"set":{"field":"event.dataset","ignore_empty_value":true,"ignore_failure":true,"value":"hydra.{{{audience}}}","media_type":"text/plain"}},
+    {"set":{"field":"event.action","ignore_failure":true,"copy_from":"msg"    }},
+    { "pipeline":    { "name": "common" } }
+  ]
+}

salt/elasticsearch/files/ingest/import.wel

@@ -1,11 +0,0 @@
-{
-  "description" : "import.wel",
-  "processors" : [
-    { "set":           { "field": "event.ingested", "value": "{{ @timestamp }}"  } },
-    { "set" :          { "field" : "@timestamp", "value" : "{{ event.created }}" } },
-    { "remove":        { "field": [ "event_record_id", "event.created" , "timestamp" , "winlog.event_data.UtcTime" ], "ignore_failure": true } },
-    { "pipeline":      { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon" } },
-    { "pipeline":      { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" } },
-    { "pipeline":      { "name": "common" } }
-  ]
-}

salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0

@@ -1,389 +0,0 @@
-{
-  "description": "Pipeline for PFsense",
-  "processors": [
-    {
-      "set": {
-        "field": "ecs.version",
-        "value": "8.10.0"
-      }
-    },
-    {
-      "set": {
-        "field": "observer.vendor",
-        "value": "netgate"
-      }
-    },
-    {
-      "set": {
-        "field": "observer.type",
-        "value": "firewall"
-      }
-    },
-    {
-      "rename": {
-        "field": "message",
-        "target_field": "event.original"
-      }
-    },
-    {
-      "set": {
-        "field": "event.kind",
-        "value": "event"
-      }
-    },
-    {
-      "set": {
-        "field": "event.timezone",
-        "value": "{{_tmp.tz_offset}}",
-        "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'"
-      }
-    },
-    {
-      "grok": {
-        "description": "Parse syslog header",
-        "field": "event.original",
-        "patterns": [
-          "^(%{ECS_SYSLOG_PRI})?%{TIMESTAMP} %{GREEDYDATA:message}"
-        ],
-        "pattern_definitions": {
-          "ECS_SYSLOG_PRI": "<%{NONNEGINT:log.syslog.priority:long}>(\\d )?",
-          "TIMESTAMP": "(?:%{BSD_TIMESTAMP_FORMAT}|%{SYSLOG_TIMESTAMP_FORMAT})",
-          "BSD_TIMESTAMP_FORMAT": "%{SYSLOGTIMESTAMP:_tmp.timestamp}(%{SPACE}%{BSD_PROCNAME}|%{SPACE}%{OBSERVER}%{SPACE}%{BSD_PROCNAME})(\\[%{POSINT:process.pid:long}\\])?:",
-          "BSD_PROCNAME": "(?:\\b%{NAME:process.name}|\\(%{NAME:process.name}\\))",
-          "NAME": "[[[:alnum:]]_-]+",
-          "SYSLOG_TIMESTAMP_FORMAT": "%{TIMESTAMP_ISO8601:_tmp.timestamp8601}%{SPACE}%{OBSERVER}%{SPACE}%{PROCESS}%{SPACE}(%{POSINT:process.pid:long}|-) - (-|%{META})",
-          "TIMESTAMP_ISO8601": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE:event.timezone}?",
-          "OBSERVER": "(?:%{IP:observer.ip}|%{HOSTNAME:observer.name})",
-          "PROCESS": "(\\(%{DATA:process.name}\\)|(?:%{UNIXPATH}*/)?%{BASEPATH:process.name})",
-          "BASEPATH": "[[[:alnum:]]_%!$@:.,+~-]+",
-          "META": "\\[[^\\]]*\\]"
-        }
-      }
-    },
-    {
-      "date": {
-        "if": "ctx._tmp.timestamp8601 != null",
-        "field": "_tmp.timestamp8601",
-        "target_field": "@timestamp",
-        "formats": [
-          "ISO8601"
-        ]
-      }
-    },
-    {
-      "date": {
-        "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null",
-        "field": "_tmp.timestamp",
-        "target_field": "@timestamp",
-        "formats": [
-          "MMM  d HH:mm:ss",
-          "MMM d HH:mm:ss",
-          "MMM dd HH:mm:ss"
-        ],
-        "timezone": "{{ event.timezone }}"
-      }
-    },
-    {
-      "grok": {
-        "description": "Set Event Provider",
-        "field": "process.name",
-        "patterns": [
-          "^%{HYPHENATED_WORDS:event.provider}"
-        ],
-        "pattern_definitions": {
-          "HYPHENATED_WORDS": "\\b[A-Za-z0-9_]+(-[A-Za-z_]+)*\\b"
-        }
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-firewall",
-        "if": "ctx.event.provider == 'filterlog'"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-openvpn",
-        "if": "ctx.event.provider == 'openvpn'"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-ipsec",
-        "if": "ctx.event.provider == 'charon'"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-dhcp",
-        "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-unbound",
-        "if": "ctx.event.provider == 'unbound'"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-haproxy",
-        "if": "ctx.event.provider == 'haproxy'"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-php-fpm",
-        "if": "ctx.event.provider == 'php-fpm'"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-squid",
-        "if": "ctx.event.provider == 'squid'"
-      }
-    },
-    {
-     "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-suricata",
-        "if": "ctx.event.provider == 'suricata'"
-      }
-    },
-    {
-      "drop": {
-        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"suricata\"].contains(ctx.event?.provider)"
-      }
-    },
-    {
-      "append": {
-        "field": "event.category",
-        "value": "network",
-        "if": "ctx.network != null"
-      }
-    },
-    {
-      "convert": {
-        "field": "source.address",
-        "target_field": "source.ip",
-        "type": "ip",
-        "ignore_failure": true,
-        "ignore_missing": true
-      }
-    },
-    {
-      "convert": {
-        "field": "destination.address",
-        "target_field": "destination.ip",
-        "type": "ip",
-        "ignore_failure": true,
-        "ignore_missing": true
-      }
-    },
-    {
-      "set": {
-        "field": "network.type",
-        "value": "ipv6",
-        "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")"
-      }
-    },
-    {
-      "set": {
-        "field": "network.type",
-        "value": "ipv4",
-        "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")"
-      }
-    },
-    {
-      "geoip": {
-        "field": "source.ip",
-        "target_field": "source.geo",
-        "ignore_missing": true
-      }
-    },
-    {
-      "geoip": {
-        "field": "destination.ip",
-        "target_field": "destination.geo",
-        "ignore_missing": true
-      }
-    },
-    {
-      "geoip": {
-        "ignore_missing": true,
-        "database_file": "GeoLite2-ASN.mmdb",
-        "field": "source.ip",
-        "target_field": "source.as",
-        "properties": [
-          "asn",
-          "organization_name"
-        ]
-      }
-    },
-    {
-      "geoip": {
-        "database_file": "GeoLite2-ASN.mmdb",
-        "field": "destination.ip",
-        "target_field": "destination.as",
-        "properties": [
-          "asn",
-          "organization_name"
-        ],
-        "ignore_missing": true
-      }
-    },
-    {
-      "rename": {
-        "field": "source.as.asn",
-        "target_field": "source.as.number",
-        "ignore_missing": true
-      }
-    },
-    {
-      "rename": {
-        "field": "source.as.organization_name",
-        "target_field": "source.as.organization.name",
-        "ignore_missing": true
-      }
-    },
-    {
-      "rename": {
-        "field": "destination.as.asn",
-        "target_field": "destination.as.number",
-        "ignore_missing": true
-      }
-    },
-    {
-      "rename": {
-        "field": "destination.as.organization_name",
-        "target_field": "destination.as.organization.name",
-        "ignore_missing": true
-      }
-    },
-    {
-      "community_id": {
-        "target_field": "network.community_id",
-        "ignore_failure": true
-      }
-    },
-    {
-      "grok": {
-        "field": "observer.ingress.interface.name",
-        "patterns": [
-          "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}"
-        ],
-        "ignore_missing": true,
-        "ignore_failure": true
-      }
-    },
-    {
-      "set": {
-        "field": "network.vlan.id",
-        "copy_from": "observer.ingress.vlan.id",
-        "ignore_empty_value": true
-      }
-    },
-    {
-      "append": {
-        "field": "related.ip",
-        "value": "{{destination.ip}}",
-        "allow_duplicates": false,
-        "if": "ctx.destination?.ip != null"
-      }
-    },
-    {
-      "append": {
-        "field": "related.ip",
-        "value": "{{source.ip}}",
-        "allow_duplicates": false,
-        "if": "ctx.source?.ip != null"
-      }
-    },
-    {
-      "append": {
-        "field": "related.ip",
-        "value": "{{source.nat.ip}}",
-        "allow_duplicates": false,
-        "if": "ctx.source?.nat?.ip != null"
-      }
-    },
-    {
-      "append": {
-        "field": "related.hosts",
-        "value": "{{destination.domain}}",
-        "if": "ctx.destination?.domain != null"
-      }
-    },
-    {
-      "append": {
-        "field": "related.user",
-        "value": "{{user.name}}",
-        "if": "ctx.user?.name != null"
-      }
-    },
-    {
-      "set": {
-        "field": "network.direction",
-        "value": "{{network.direction}}bound",
-        "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/"
-      }
-    },
-    {
-      "remove": {
-        "field": [
-          "_tmp"
-        ],
-        "ignore_failure": true
-      }
-    },
-    {
-      "script": {
-        "lang": "painless",
-        "description": "This script processor iterates over the whole document to remove fields with null values.",
-        "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
-      }
-    },
-    {
-      "remove": {
-        "field": "event.original",
-        "if": "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))",
-        "ignore_failure": true,
-        "ignore_missing": true
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log@custom",
-        "ignore_missing_pipeline": true
-      }
-    }
-  ],
-  "on_failure": [
-    {
-      "remove": {
-        "field": [
-          "_tmp"
-        ],
-        "ignore_failure": true
-      }
-    },
-    {
-      "set": {
-        "field": "event.kind",
-        "value": "pipeline_error"
-      }
-    },
-    {
-      "append": {
-        "field": "error.message",
-        "value": "{{{ _ingest.on_failure_message }}}"
-      }
-    }
-  ],
-  "_meta": {
-    "managed_by": "fleet",
-    "managed": true,
-    "package": {
-      "name": "pfsense"
-    }
-  }
-}

salt/elasticsearch/files/ingest/logs-pfsense.log-1.23.0

@@ -1,10 +1,17 @@
 {
   "description": "Pipeline for PFsense",
+  "_meta": {
+    "managed_by": "fleet",
+    "managed": true,
+    "package": {
+      "name": "pfsense"
+    }
+  },
   "processors": [
     {
       "set": {
         "field": "ecs.version",
-        "value": "8.11.0"
+        "value": "8.17.0"
       }
     },
     {
@@ -36,7 +43,7 @@
     {
       "set": {
         "field": "event.timezone",
-        "value": "{{_tmp.tz_offset}}",
+        "value": "{{{_tmp.tz_offset}}}",
         "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'"
       }
     },
@@ -83,7 +90,7 @@
           "MMM d HH:mm:ss",
           "MMM dd HH:mm:ss"
         ],
-        "timezone": "{{ event.timezone }}"
+        "timezone": "{{{ event.timezone }}}"
       }
     },
     {
@@ -100,61 +107,67 @@
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-firewall",
+        "name": "logs-pfsense.log-1.23.0-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-openvpn",
+        "name": "logs-pfsense.log-1.23.0-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-ipsec",
+        "name": "logs-pfsense.log-1.23.0-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-dhcp",
+        "name": "logs-pfsense.log-1.23.0-dhcp",
         "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-unbound",
+        "name": "logs-pfsense.log-1.23.0-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-haproxy",
+        "name": "logs-pfsense.log-1.23.0-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-php-fpm",
+        "name": "logs-pfsense.log-1.23.0-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-squid",
+        "name": "logs-pfsense.log-1.23.0-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
-     "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-suricata",
+      "pipeline": {
+        "name": "logs-pfsense.log-1.23.0-snort",
+        "if": "ctx.event.provider == 'snort'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.23.0-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },
     {
       "drop": {
-        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"suricata\"].contains(ctx.event?.provider)"
+        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
       }
     },
     {
@@ -288,7 +301,7 @@
     {
       "append": {
         "field": "related.ip",
-        "value": "{{destination.ip}}",
+        "value": "{{{destination.ip}}}",
         "allow_duplicates": false,
         "if": "ctx.destination?.ip != null"
       }
@@ -296,7 +309,7 @@
     {
       "append": {
         "field": "related.ip",
-        "value": "{{source.ip}}",
+        "value": "{{{source.ip}}}",
         "allow_duplicates": false,
         "if": "ctx.source?.ip != null"
       }
@@ -304,7 +317,7 @@
     {
       "append": {
         "field": "related.ip",
-        "value": "{{source.nat.ip}}",
+        "value": "{{{source.nat.ip}}}",
         "allow_duplicates": false,
         "if": "ctx.source?.nat?.ip != null"
       }
@@ -312,21 +325,21 @@
     {
       "append": {
         "field": "related.hosts",
-        "value": "{{destination.domain}}",
+        "value": "{{{destination.domain}}}",
         "if": "ctx.destination?.domain != null"
       }
     },
     {
       "append": {
         "field": "related.user",
-        "value": "{{user.name}}",
+        "value": "{{{user.name}}}",
         "if": "ctx.user?.name != null"
       }
     },
     {
       "set": {
         "field": "network.direction",
-        "value": "{{network.direction}}bound",
+        "value": "{{{network.direction}}}bound",
         "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/"
       }
     },
@@ -345,14 +358,6 @@
         "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
       }
     },
-    {
-      "remove": {
-        "field": "event.original",
-        "if": "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))",
-        "ignore_failure": true,
-        "ignore_missing": true
-      }
-    },
     {
       "pipeline": {
         "name": "global@custom",
@@ -403,12 +408,5 @@
         "value": "{{{ _ingest.on_failure_message }}}"
       }
     }
-  ],
-  "_meta": {
-    "managed_by": "fleet",
-    "managed": true,
-    "package": {
-      "name": "pfsense"
-    }
-  }
-}
+  ]
+}

salt/elasticsearch/files/ingest/suricata.alert

@@ -1,7 +1,7 @@
 {
   "description" : "suricata.alert",
   "processors" : [
-    { "set":   { "field": "_index", "value": "logs-suricata.alerts-so" } },
+    { "set":   { "if": "ctx.event?.imported != true", "field": "_index", "value": "logs-suricata.alerts-so" } },
     { "set":   { "field": "tags","value": "alert" }},
     { "rename":{ "field": "message2.alert",	"target_field": "rule",	"ignore_failure": true 	} },
     { "rename":{ "field": "rule.signature",     "target_field": "rule.name", "ignore_failure": true  } },
@@ -9,6 +9,7 @@
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.uuid", "ignore_failure": true  } },
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.signature", "ignore_failure": true  } },
     { "rename":{ "field": "message2.payload_printable",     "target_field": "network.data.decoded", "ignore_failure": true  } },
+    { "dissect": { "field": "rule.rule", "pattern": "%{?prefix}content:\"%{dns.query_name}\"%{?remainder}", "ignore_missing": true, "ignore_failure": true } },
     { "pipeline": { "name": "common.nids" } }
   ]
-}
+}

salt/elasticsearch/files/ingest/suricata.common

@@ -18,6 +18,13 @@
     { "set":    { "field": "event.ingested",            "value": "{{@timestamp}}"                                       } },
     { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
     { "remove":{ "field": "agent",                                                              "ignore_failure": true  } },
+    {"append":{"field":"related.ip","value":["{{source.ip}}","{{destination.ip}}"],"allow_duplicates":false,"ignore_failure":true}},
+    {
+      "script": {
+        "source": "boolean isPrivate(def ip) { if (ip == null) return false; int dot1 = ip.indexOf('.'); if (dot1 == -1) return false; int dot2 = ip.indexOf('.', dot1 + 1); if (dot2 == -1) return false; int first = Integer.parseInt(ip.substring(0, dot1)); if (first == 10) return true; if (first == 192 && ip.startsWith('168.', dot1 + 1)) return true; if (first == 172) { int second = Integer.parseInt(ip.substring(dot1 + 1, dot2)); return second >= 16 && second <= 31; } return false; } String[] fields = new String[] {\"source\", \"destination\"}; for (int i = 0; i < fields.length; i++) { def field = fields[i]; def ip = ctx[field]?.ip; if (ip != null) { if (ctx.network == null) ctx.network = new HashMap(); if (isPrivate(ip)) { if (ctx.network.private_ip == null) ctx.network.private_ip = new ArrayList(); if (!ctx.network.private_ip.contains(ip)) ctx.network.private_ip.add(ip); } else { if (ctx.network.public_ip == null) ctx.network.public_ip = new ArrayList(); if (!ctx.network.public_ip.contains(ip)) ctx.network.public_ip.add(ip); } } }",
+        "ignore_failure": false
+      }
+    },
     { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}" } }
   ]
 }

salt/elasticsearch/files/ingest/zeek.common

@@ -12,12 +12,14 @@
     { "rename":         { "field": "message2.id.orig_p",        "target_field": "source.port",          "ignore_missing": true  } },
     { "rename":         { "field": "message2.id.resp_h",        "target_field": "destination.ip",       "ignore_missing": true  } },
     { "rename":         { "field": "message2.id.resp_p",        "target_field": "destination.port",     "ignore_missing": true  } },
-    { "community_id": {} },
+    { "rename":         { "field": "message2.community_id",     "target_field": "network.community_id", "ignore_missing": true  } },
+    { "community_id":   { "if":    "ctx.network?.community_id == null"     } },
     { "set":         { "if": "ctx.source?.ip != null", "field": "client.ip",        "value": "{{source.ip}}"           } },
     { "set":         { "if": "ctx.source?.port != null", "field": "client.port",        "value": "{{source.port}}"       } },
     { "set":         { "if": "ctx.destination?.ip != null", "field": "server.ip",        "value": "{{destination.ip}}"  } },
     { "set":         { "if": "ctx.destination?.port != null", "field": "server.port",        "value": "{{destination.port}}"  } },
     { "set":         { "field": "observer.name",        "value": "{{agent.name}}"  } },
+    { "append": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"openvpn\")","field": "tags","value": ["{{network.protocol}}"],"allow_duplicates": false,"ignore_failure": true}},
     { "date":		{ "field": "message2.ts",	"target_field": "@timestamp",	"formats": ["ISO8601", "UNIX"], "ignore_failure": true	} },
     { "remove":		{ "field": ["agent"],	"ignore_failure": true									} },
     { "pipeline":	{ "name": "common" 													} }

salt/elasticsearch/files/ingest/zeek.conn

@@ -24,6 +24,10 @@
     { "rename": 	{ "field": "message2.resp_cc", 		"target_field": "server.country_code",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.sensorname", 	"target_field": "observer.name",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.vlan", 		"target_field": "network.vlan.id",		"ignore_missing": true 	} },
+    { "rename":   { "field": "message2.ja4l",      "target_field": "hash.ja4l",         "ignore_missing" : true,        "if": "ctx.message2?.ja4l != null && ctx.message2.ja4l.length() > 0" }},
+    { "rename":   { "field": "message2.ja4ls",     "target_field": "hash.ja4ls",        "ignore_missing" : true,        "if": "ctx.message2?.ja4ls != null && ctx.message2.ja4ls.length() > 0" }},
+    { "rename":   { "field": "message2.ja4t",      "target_field": "hash.ja4t",         "ignore_missing" : true,        "if": "ctx.message2?.ja4t != null && ctx.message2.ja4t.length() > 0" }},
+    { "rename":   { "field": "message2.ja4ts",      "target_field": "hash.ja4ts",         "ignore_missing" : true,      "if": "ctx.message2?.ja4ts != null && ctx.message2.ja4ts.length() > 0" }},
     { "script":         { "lang": "painless", "source": "ctx.network.bytes = (ctx.client.bytes + ctx.server.bytes)", "ignore_failure": true } },
     { "set": { "if": "ctx.connection?.state == 'S0'",    "field": "connection.state_description", "value": "Connection attempt seen, no reply" } },
     { "set": { "if": "ctx.connection?.state == 'S1'",    "field": "connection.state_description", "value": "Connection established, not terminated" } },
@@ -38,6 +42,8 @@
     { "set": { "if": "ctx.connection?.state == 'SH'",    "field": "connection.state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } },
     { "set": { "if": "ctx.connection?.state == 'SHR'",   "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } },
     { "set": { "if": "ctx.connection?.state == 'OTH'",   "field": "connection.state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } },
+    { "set": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"ipsec\")", "field": "network.protocol", "value": "ipsec"}},
+    { "set": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"openvpn\")", "field": "network.protocol", "value": "openvpn"}},
     { "pipeline": 	{ "name": "zeek.common" } }
   ]
 }

salt/elasticsearch/files/ingest/zeek.dns

@@ -20,7 +20,8 @@
     { "rename": 	{ "field": "message2.RD", 		"target_field": "dns.recursion.desired",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.RA", 		"target_field": "dns.recursion.available",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.Z",		"target_field": "dns.reserved",			"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.answers", 		"target_field": "dns.answers.name",		"ignore_missing": true 	} },
+    { "rename":  	{ "field": "message2.answers", 		"target_field": "dns.answers.name", 		"ignore_missing": true  	} },
+    { "script": { "lang": "painless", "if": "ctx.dns != null && ctx.dns.answers != null && ctx.dns.answers.name != null", "source": "def ips = []; for (item in ctx.dns.answers.name) { if (item =~ /^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}$/ || item =~ /^([a-fA-F0-9:]+:+)+[a-fA-F0-9]+$/) { ips.add(item); } } ctx.dns.resolved_ip = ips;" } },
     { "rename": 	{ "field": "message2.TTLs", 		"target_field": "dns.ttls",			"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.rejected", 	"target_field": "dns.query.rejected",		"ignore_missing": true 	} },
     { "script": 	{ "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()",	"ignore_failure": true  } },
@@ -28,4 +29,4 @@
     { "pipeline": { "if": "ctx.dns?.query?.name != null && ctx.dns.query.name.contains('.')", "name": "dns.tld" } },
     { "pipeline":       { "name": "zeek.common"											} }
   ]
-}
+}

salt/elasticsearch/files/ingest/zeek.http

@@ -27,6 +27,7 @@
     { "rename": 	{ "field": "message2.resp_fuids",	"target_field": "log.id.resp_fuids",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.resp_filenames",	"target_field": "file.resp_filenames",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.resp_mime_types",	"target_field": "file.resp_mime_types",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.ja4h",	"target_field": "hash.ja4h",	"ignore_missing": true, "if": "ctx?.message2?.ja4h != null && ctx.message2.ja4h.length() > 0" 	} },
     { "script":	{ "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", 			"ignore_failure": true 	} },
     { "script":	{ "lang": "painless", "source": "ctx.useragent_length = ctx.useragent.length()",	"ignore_failure": true 	} },
     { "script":	{ "lang": "painless", "source": "ctx.virtual_host_length = ctx.virtual_host.length()", 	"ignore_failure": true	} },

salt/elasticsearch/files/ingest/zeek.http2

@@ -0,0 +1,38 @@
+{
+  "description" : "zeek.http2",
+  "processors" : [
+    { "set":      { "field": "event.dataset",                   "value": "http2" } },
+    { "set":      { "field": "network.transport",               "value": "tcp"  } },
+    { "json":     { "field": "message",                         "target_field": "message2",                       "ignore_failure": true  } },
+    { "rename": 	{ "field": "message2.trans_depth",	          "target_field": "http.trans_depth",		        "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.method", 		            "target_field": "http.method",		            "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.host", 		              "target_field": "http.virtual_host",		      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.uri", 		                "target_field": "http.uri",			              "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.referrer", 	            "target_field": "http.referrer",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.version", 		            "target_field": "http.version",		            "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.user_agent", 	          "target_field": "http.useragent",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.request_body_len",       "target_field": "http.request.body.length",	  "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.response_body_len",      "target_field": "http.response.body.length",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.status_code",	          "target_field": "http.status_code",		        "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.status_msg", 	          "target_field": "http.status_message",	      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.info_code", 	            "target_field": "http.info_code",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.info_msg", 	            "target_field": "http.info_message",		      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.username", 	            "target_field": "http.user",			            "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.password", 	            "target_field": "http.password",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proxied", 		            "target_field": "http.proxied",		            "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.orig_fuids",	            "target_field": "log.id.orig_fuids",		      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.orig_filenames",	        "target_field": "file.orig_filenames",	      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.orig_mime_types",	      "target_field": "file.orig_mime_types",	      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.resp_fuids",	            "target_field": "log.id.resp_fuids",		      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.resp_filenames",	        "target_field": "file.resp_filenames",	      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.resp_mime_types",	      "target_field": "file.resp_mime_types",	      "ignore_missing": true 	} },
+    { "rename":   { "field": "message2.stream_id",              "target_field": "http2.stream_id",            "ignore_missing": true  } },
+    { "rename":   { "field": "message2.ja4h",                   "target_field": "hash.ja4h",                  "ignore_missing": true,   "if": "ctx?.message2?.ja4h != null && ctx.message2.ja4h.length() > 0"  } },
+    { "remove":		{ "field": "message2.tags",		                                                                  "ignore_failure": true } },
+    { "remove":   { "field": ["host"],                                                                            "ignore_failure": true } },
+    { "script":	{ "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", 			                        "ignore_failure": true 	} },
+    { "script":	{ "lang": "painless", "source": "ctx.useragent_length = ctx.useragent.length()",	                "ignore_failure": true 	} },
+    { "script":	{ "lang": "painless", "source": "ctx.virtual_host_length = ctx.virtual_host.length()", 	          "ignore_failure": true	} },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.ipsec

@@ -0,0 +1,38 @@
+{
+  "description": "zeek.ipsec",
+  "processors": [
+    {"set": { "field": "event.dataset","value": "ipsec"}},
+    {"json": { "field": "message","target_field": "message2","ignore_failure": true}},
+    {"rename": {"field": "message2.initiator_spi","target_field": "ipsec.initiator_spi","ignore_missing": true}},
+    {"rename": {"field": "message2.responder_spi","target_field": "ipsec.responder_spi","ignore_missing": true}},
+    {"rename": {"field": "message2.maj_ver","target_field": "ipsec.maj_version","ignore_missing": true}},
+    {"rename": {"field": "message2.min_ver","target_field": "ipsec.min_version","ignore_missing": true}},
+    {"set": {"ignore_failure": true,"field": "ipsec.version","value": "{{ipsec.maj_version}}.{{ipsec.min_version}}"}},
+    {"rename": {"field": "message2.exchange_type","target_field": "ipsec.exchange_type","ignore_missing": true}},
+    {"rename": {"field": "message2.flag_e","target_field": "ipsec.flag_e","ignore_missing": true}},
+    {"rename": {"field": "message2.flag_c","target_field": "ipsec.flag_c","ignore_missing": true}},
+    {"rename": {"field": "message2.flag_a","target_field": "ipsec.flag_a","ignore_missing": true}},
+    {"rename": {"field": "message2.flag_i","target_field": "ipsec.flag_i","ignore_missing": true}},
+    {"rename": {"field": "message2.flag_v","target_field": "ipsec.flag_v","ignore_missing": true}},
+    {"rename": {"field": "message2.flag_r","target_field": "ipsec.flag_r","ignore_missing": true}},
+    {"rename": {"field": "message2.message_id","target_field": "ipsec.message_id","ignore_missing": true}},
+    {"rename": {"field": "message2.vendor_ids","target_field": "ipsec.vendor_ids","ignore_missing": true}},
+    {"rename": {"field": "message2.notify_messages","target_field": "ipsec.notify_messages","ignore_missing": true}},
+    {"rename": {"field": "message2.transforms","target_field": "ipsec.transforms","ignore_missing": true}},
+    {"rename": {"field": "message2.ke_dh_groups","target_field": "ipsec.ke_dh_groups","ignore_missing": true}},
+    {"rename": {"field": "message2.proposals","target_field": "ipsec.proposals","ignore_missing": true}},
+    {"rename": {"field": "message2.certificates","target_field": "ipsec.certificates","ignore_missing": true}},
+    {"rename": {"field": "message2.transform_attributes","target_field": "ipsec.transform_attributes","ignore_missing": true}},
+    {"rename": {"field": "message2.length","target_field": "ipsec.length","ignore_missing": true}},
+    {"rename": {"field": "message2.hash","target_field": "ipsec.hash","ignore_missing": true}},
+    {"rename": {"field": "message2.doi","target_field": "ipsec.doi","ignore_missing": true}},
+    {"rename": {"field": "message2.situation","target_field": "ipsec.situation","ignore_missing": true}},
+    {"script": {
+      "lang": "painless",
+      "description": "Remove ipsec fields with empty arrays",
+      "source": "if (ctx.containsKey('ipsec') && ctx.ipsec instanceof Map) {\n        for (String field : ['certificates', 'ke_dh_groups', 'notify_messages', 'proposals', 'transforms', 'transform_attributes', 'vendor_ids']) {\n          if (ctx.ipsec[field] instanceof List && ctx.ipsec[field].isEmpty()) {\n            ctx.ipsec.remove(field);\n          }\n        }\n      }",
+      "ignore_failure": true
+      }},
+    {"pipeline": {"name": "zeek.common"}}
+  ]
+}

salt/elasticsearch/files/ingest/zeek.ja4ssh

@@ -0,0 +1,10 @@
+{
+  "description": "zeek.ja4ssh",
+  "processors": [
+    {"set": {"field": "event.dataset","value": "ja4ssh"}},
+    {"remove": {"field": "host","ignore_missing": true,"ignore_failure": true}},
+    {"json": {"field": "message","target_field": "message2","ignore_failure": true}},
+    {"rename": {"field": "message2.ja4ssh", "target_field": "hash.ja4ssh", "ignore_missing": true, "if": "ctx?.message2?.ja4ssh != null && ctx.message2.ja4ssh.length() > 0" }},
+    {"pipeline": {"name": "zeek.common"}}
+  ]
+}

salt/elasticsearch/files/ingest/zeek.ldap

@@ -0,0 +1,25 @@
+{
+    "description": "zeek.ldap",
+    "processors": [
+        {"set":      {"field": "event.dataset",                 "value": "ldap"}},
+        {"json":     {"field": "message",                       "target_field": "message2",                     "ignore_failure": true}},
+        {"rename":   {"field": "message2.message_id",           "target_field": "ldap.message_id",              "ignore_missing": true}},
+        {"rename":   {"field": "message2.opcode",               "target_field": "ldap.opcode",                  "ignore_missing": true}},
+        {"rename":   {"field": "message2.result",               "target_field": "ldap.result",                  "ignore_missing": true}},
+        {"rename":   {"field": "message2.diagnostic_message",   "target_field": "ldap.diagnostic_message",      "ignore_missing": true}},
+        {"rename":   {"field": "message2.version",              "target_field": "ldap.version",                 "ignore_missing": true}},
+        {"rename":   {"field": "message2.object",               "target_field": "ldap.object",                  "ignore_missing": true}},
+        {"rename":   {"field": "message2.argument",             "target_field": "ldap.argument",                "ignore_missing": true}},
+        {"rename":   {"field": "message2.scope",                "target_field": "ldap_search.scope",            "ignore_missing":true}},
+        {"rename":   {"field": "message2.deref_aliases",        "target_field": "ldap_search.deref_aliases",    "ignore_missing":true}},
+        {"rename":   {"field": "message2.base_object",          "target_field": "ldap.object",                  "ignore_missing":true}},
+        {"rename":   {"field": "message2.result_count",         "target_field": "ldap_search.result_count",     "ignore_missing":true}},
+        {"rename":   {"field": "message2.filter",               "target_field": "ldap_search.filter",           "ignore_missing":true}},
+        {"rename":   {"field": "message2.attributes",           "target_field": "ldap_search.attributes",       "ignore_missing":true}},
+        {"script":   {"source": "if (ctx.containsKey('ldap') && ctx.ldap.containsKey('diagnostic_message') && ctx.ldap.diagnostic_message != null) {\n          String message = ctx.ldap.diagnostic_message;\n\n          // get user and property from SASL success\n          if (message.toLowerCase().contains(\"sasl(0): successful result\")) {\n            Pattern pattern = /user:\\s*([^ ]+)\\s*property:\\s*([^ ]+)/i;\n            Matcher matcher = pattern.matcher(message);\n            if (matcher.find()) {\n              ctx.ldap.user_email = matcher.group(1);  // Extract user email\n              ctx.ldap.property = matcher.group(2);    // Extract property\n            }\n          }\n          if (message.toLowerCase().contains(\"ldaperr:\")) {\n            Pattern pattern = /comment:\\s*([^,]+)/i;\n            Matcher matcher = pattern.matcher(message);\n\n            if (matcher.find()) {\n              ctx.ldap.comment = matcher.group(1);\n            }\n          }\n        }","ignore_failure": true}},
+        {"script":   {"source": "if (ctx.containsKey('ldap') && ctx.ldap.containsKey('object') && ctx.ldap.object != null) {\n    String message = ctx.ldap.object;\n\n    // parse common name from ldap object\n    if (message.toLowerCase().contains(\"cn=\")) {\n        Pattern pattern = /cn=([^,]+)/i;\n        Matcher matcher = pattern.matcher(message);\n        if (matcher.find()) {\n            ctx.ldap.common_name = matcher.group(1);  // Extract CN\n        }\n    }\n    // build domain from ldap object\n    if (message.toLowerCase().contains(\"dc=\")) {\n        Pattern dcPattern = /dc=([^,]+)/i;\n        Matcher dcMatcher = dcPattern.matcher(message);\n\n        StringBuilder domainBuilder = new StringBuilder();\n        while (dcMatcher.find()) {\n            if (domainBuilder.length() > 0 ){\n                domainBuilder.append(\".\");\n            }\n            domainBuilder.append(dcMatcher.group(1));\n        }\n        if (domainBuilder.length() > 0) {\n            ctx.ldap.domain = domainBuilder.toString();\n        }\n    }\n    // create list of any organizational units from ldap object\n    if (message.toLowerCase().contains(\"ou=\")) {\n        Pattern ouPattern = /ou=([^,]+)/i;\n        Matcher ouMatcher = ouPattern.matcher(message);\n        ctx.ldap.organizational_unit = [];\n\n        while (ouMatcher.find()) {\n            ctx.ldap.organizational_unit.add(ouMatcher.group(1));\n        }\n        if(ctx.ldap.organizational_unit.isEmpty()) {\n          ctx.remove(\"ldap.organizational_unit\");\n        }\n    }\n}\n","ignore_failure": true}},
+        {"remove":   {"field": "message2.tags","ignore_failure": true}},
+        {"remove":   {"field": ["host"],"ignore_failure": true}},
+        {"pipeline": {"name": "zeek.common"}}
+    ]
+}

salt/elasticsearch/files/ingest/zeek.ldap_search

@@ -0,0 +1,25 @@
+{
+    "description":"zeek.ldap_search",
+    "processors":[
+        {"set":         {"field": "event.dataset",                 "value":"ldap_search"}},
+        {"json":        {"field": "message",                       "target_field": "message2",                     "ignore_failure": true}},
+        {"rename":      {"field": "message2.message_id",           "target_field": "ldap.message_id",              "ignore_missing": true}},
+        {"rename":      {"field": "message2.opcode",               "target_field": "ldap.opcode",                  "ignore_missing": true}},
+        {"rename":      {"field": "message2.result",               "target_field": "ldap.result",                  "ignore_missing": true}},
+        {"rename":      {"field": "message2.diagnostic_message",   "target_field": "ldap.diagnostic_message",      "ignore_missing": true}},
+        {"rename":      {"field": "message2.version",              "target_field": "ldap.version",                 "ignore_missing": true}},
+        {"rename":      {"field": "message2.object",               "target_field": "ldap.object",                  "ignore_missing": true}},
+        {"rename":      {"field": "message2.argument",             "target_field": "ldap.argument",                "ignore_missing": true}},
+        {"rename":      {"field": "message2.scope",                "target_field": "ldap_search.scope",            "ignore_missing":true}},
+        {"rename":      {"field": "message2.deref_aliases",        "target_field": "ldap_search.deref_aliases",    "ignore_missing":true}},
+        {"rename":      {"field": "message2.base_object",          "target_field": "ldap.object",                  "ignore_missing":true}},
+        {"rename":      {"field": "message2.result_count",         "target_field": "ldap_search.result_count",     "ignore_missing":true}},
+        {"rename":      {"field": "message2.filter",               "target_field": "ldap_search.filter",           "ignore_missing":true}},
+        {"rename":      {"field": "message2.attributes",           "target_field": "ldap_search.attributes",       "ignore_missing":true}},
+        {"script":      {"source": "if (ctx.containsKey('ldap') && ctx.ldap.containsKey('diagnostic_message') && ctx.ldap.diagnostic_message != null) {\n          String message = ctx.ldap.diagnostic_message;\n\n          // get user and property from SASL success\n          if (message.toLowerCase().contains(\"sasl(0): successful result\")) {\n            Pattern pattern = /user:\\s*([^ ]+)\\s*property:\\s*([^ ]+)/i;\n            Matcher matcher = pattern.matcher(message);\n            if (matcher.find()) {\n              ctx.ldap.user_email = matcher.group(1);  // Extract user email\n              ctx.ldap.property = matcher.group(2);    // Extract property\n            }\n          }\n          if (message.toLowerCase().contains(\"ldaperr:\")) {\n            Pattern pattern = /comment:\\s*([^,]+)/i;\n            Matcher matcher = pattern.matcher(message);\n\n            if (matcher.find()) {\n              ctx.ldap.comment = matcher.group(1);\n            }\n          }\n        }","ignore_failure": true}},
+        {"script":      {"source": "if (ctx.containsKey('ldap') && ctx.ldap.containsKey('object') && ctx.ldap.object != null) {\n    String message = ctx.ldap.object;\n\n    // parse common name from ldap object\n    if (message.toLowerCase().contains(\"cn=\")) {\n        Pattern pattern = /cn=([^,]+)/i;\n        Matcher matcher = pattern.matcher(message);\n        if (matcher.find()) {\n            ctx.ldap.common_name = matcher.group(1);  // Extract CN\n        }\n    }\n    // build domain from ldap object\n    if (message.toLowerCase().contains(\"dc=\")) {\n        Pattern dcPattern = /dc=([^,]+)/i;\n        Matcher dcMatcher = dcPattern.matcher(message);\n\n        StringBuilder domainBuilder = new StringBuilder();\n        while (dcMatcher.find()) {\n            if (domainBuilder.length() > 0 ){\n                domainBuilder.append(\".\");\n            }\n            domainBuilder.append(dcMatcher.group(1));\n        }\n        if (domainBuilder.length() > 0) {\n            ctx.ldap.domain = domainBuilder.toString();\n        }\n    }\n    // create list of any organizational units from ldap object\n    if (message.toLowerCase().contains(\"ou=\")) {\n        Pattern ouPattern = /ou=([^,]+)/i;\n        Matcher ouMatcher = ouPattern.matcher(message);\n        ctx.ldap.organizational_unit = [];\n\n        while (ouMatcher.find()) {\n            ctx.ldap.organizational_unit.add(ouMatcher.group(1));\n        }\n        if(ctx.ldap.organizational_unit.isEmpty()) {\n          ctx.remove(\"ldap.organizational_unit\");\n        }\n    }\n}\n","ignore_failure": true}},
+        {"remove":      {"field": "message2.tags",                 "ignore_failure": true}},
+        {"remove":      {"field": ["host"],                        "ignore_failure": true}},
+        {"pipeline":    {"name": "zeek.common"}}
+    ]
+}

salt/elasticsearch/files/ingest/zeek.ntp

@@ -0,0 +1,16 @@
+{
+  "description" : "zeek.ntp",
+  "processors":[
+    {"set":     {"field":"event.dataset",       "value":"ntp",                       "ignore_failure":true}},
+    {"json":    {"field":"message",             "target_field":"message2",           "ignore_failure":true}},
+    {"rename":  {"field":"message2.version",    "target_field":"ntp.version",        "ignore_missing":true}},
+    {"rename":  {"field":"message2.mode",       "target_field":"ntp.mode",           "ignore_missing":true}},
+    {"rename":  {"field":"message2.poll",       "target_field":"ntp.poll",           "ignore_missing":true}},
+    {"rename":  {"field":"message2.precision",  "target_field":"ntp.precision",      "ignore_missing":true}},
+    {"rename":  {"field":"message2.org_time",   "target_field":"ntp.org_time",       "ignore_missing":true}},
+    {"rename":  {"field":"message2.xmt_time",   "target_field":"ntp.xmt_time",       "ignore_missing":true}},
+    {"date":    {"field":"ntp.org_time",        "target_field":"ntp.org_time",  "formats":["UNIX", "UNIX_MS"], "ignore_failure": true, "if":"ctx?.ntp?.org_time != null"}},
+    {"date":    {"field":"ntp.xmt_time",        "target_field":"ntp.xmt_time",  "formats":["UNIX", "UNIX_MS"], "ignore_failure": true, "if":"ctx?.ntp?.xmt_time != null"}},
+    {"pipeline":{"name":"zeek.common"}}
+  ]
+}

salt/elasticsearch/files/ingest/zeek.quic

@@ -0,0 +1,18 @@
+{
+  "description" : "zeek.quic",
+  "processors" : [
+    { "set":      { "field": "event.dataset",                           "value": "quic" } },
+    { "set":      { "field": "network.transport",                       "value": "udp"  } },
+    { "json":     { "field": "message",                                 "target_field": "message2",                       "ignore_failure": true    } },
+    { "rename": 	{ "field": "message2.version",	                    "target_field": "quic.version",		              "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_initial_dcid",	        "target_field": "quic.client_initial_dcid",		  "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_scid",	                "target_field": "quic.client_scid",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_scid",	                "target_field": "quic.server_scid",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_name",	                "target_field": "quic.server_name",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_protocol",	            "target_field": "quic.client_protocol",		      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.history",	                    "target_field": "quic.history",                   "ignore_missing": true 	} },
+    { "remove":		{ "field": "message2.tags",		                                                                      "ignore_failure": true    } },
+    { "remove":   { "field": ["host"],                                                                                    "ignore_failure": true    } },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.software

@@ -11,7 +11,7 @@
     { "dot_expander": 	{ "field": "version.minor2", 		"path": "message2", 			"ignore_failure": true 	} },
     { "rename": 	{ "field": "message2.version.minor2", 	"target_field": "software.version.minor2",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "version.minor3", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.version.minor3", 	"target_field": "version.minor3",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.version.minor3", 	"target_field": "software.version.minor3",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "version.addl", 		"path": "message2", 			"ignore_failure": true 	} },
     { "rename": 	{ "field": "message2.version.addl", 	"target_field": "software.version.additional_info",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.host", 		"target_field": "source.ip",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/zeek.ssl

@@ -23,6 +23,8 @@
     { "rename":         { "field": "message2.validation_status","target_field": "ssl.validation_status",        "ignore_missing": true  } },
     { "rename":         { "field": "message2.ja3",              "target_field": "hash.ja3",                     "ignore_missing": true  } },
     { "rename":         { "field": "message2.ja3s",             "target_field": "hash.ja3s",                    "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ja4",             "target_field": "hash.ja4",                    "ignore_missing": true,         "if": "ctx?.message2?.ja4 != null && ctx.message2.ja4.length() > 0"  } },
+    { "rename":         { "field": "message2.ja4s",             "target_field": "hash.ja4s",                    "ignore_missing": true,       "if": "ctx?.message2?.ja4s != null && ctx.message2.ja4s.length() > 0"  } },
     { "foreach":
       {
         "if": "ctx?.tls?.client?.hash?.sha256 !=null",

salt/elasticsearch/files/ingest/zeek.traceroute

@@ -0,0 +1,10 @@
+{
+  "description":"zeek.traceroute",
+  "processors":[
+    {"set":         {"field":"event.dataset",       "value":"traceroute" }},
+    {"json":        {"field":"message",             "target_field":"message2" }},
+    {"rename":      {"field":"message2.src",        "target_field":"source.ip",         "ignore_missing":true,"ignore_failure":true}},
+    {"rename":      {"field":"message2.dst",        "target_field":"destination.ip",    "ignore_missing":true,"ignore_failure":true}},
+    {"pipeline":    {"name":"zeek.common"}}
+  ]
+}

salt/elasticsearch/files/ingest/zeek.x509

@@ -42,6 +42,7 @@
     { "dot_expander":   { "field": "basic_constraints.path_length",             "path": "message2",                                     "ignore_failure": true  } },
     { "rename":         { "field": "message2.basic_constraints.path_length",    "target_field": "x509.basic_constraints.path_length",   "ignore_missing": true  } },
     { "rename":         { "field": "message2.fingerprint",                      "target_field": "hash.sha256",   "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ja4x",                             "target_field": "hash.ja4x",   "ignore_missing": true, "if": "ctx?.message2?.ja4x != null && ctx.message2.ja4x.length() > 0"  } },
     { "pipeline":       { "name": "zeek.common_ssl"                                                                                                             } }
   ]
 }

salt/elasticsearch/soc_elasticsearch.yaml

@@ -12,7 +12,7 @@ elasticsearch:
     description: Specify the memory heap size in (m)egabytes for Elasticsearch.
     helpLink: elasticsearch.html
   index_clean:
-    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings.
+    description: Determines if indices should be considered for deletion by available disk space in the cluster. Otherwise, indices will only be deleted by the age defined in the ILM settings. This setting only applies to EVAL, STANDALONE, and HEAVY NODE installations. Other installations can only use ILM settings.
     forcedType: bool
     helpLink: elasticsearch.html
   retention: 
@@ -77,6 +77,13 @@ elasticsearch:
     custom008: *pipelines
     custom009: *pipelines
     custom010: *pipelines
+  managed_integrations:
+    description: List of integrations to add into SOC config UI. Enter the full or partial integration name. Eg. 1password, 1pass
+    forcedType: "[]string"
+    multiline: True
+    global: True
+    advanced: True
+    helpLink: elasticsearch.html
   index_settings:
     global_overrides:
       index_template:
@@ -126,7 +133,7 @@ elasticsearch:
                   helpLink: elasticsearch.html
           cold:
             min_age:
-              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
               regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
@@ -139,10 +146,11 @@ elasticsearch:
                   helpLink: elasticsearch.html
           warm:
             min_age: 
-              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier.
+              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
               regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
+              helpLink: elasticsearch.html
             actions:
               set_priority:
                 priority:
@@ -152,7 +160,7 @@ elasticsearch:
                   helpLink: elasticsearch.html
           delete:
             min_age:
-              description: Minimum age of index. ex. 90d - This determines when the index should be deleted.
+              description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
               regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
@@ -166,7 +174,7 @@ elasticsearch:
       index_template:
         index_patterns:
           description: Patterns for matching multiple indices or tables.
-          forceType: "[]string"
+          forcedType: "[]string"
           multiline: True
           global: True
           advanced: True
@@ -281,7 +289,7 @@ elasticsearch:
                   helpLink: elasticsearch.html
           warm:
             min_age:
-              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier.
+              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
               regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
@@ -308,7 +316,7 @@ elasticsearch:
                   helpLink: elasticsearch.html
           cold:
             min_age:
-              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
               regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
@@ -324,7 +332,7 @@ elasticsearch:
                   helpLink: elasticsearch.html
           delete:
             min_age:
-              description: Minimum age of index. This determines when the index should be deleted.
+              description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
               regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
@@ -358,156 +366,13 @@ elasticsearch:
     so-logs-windows_x_powershell_operational: *indexSettings
     so-logs-windows_x_sysmon_operational: *indexSettings
     so-logs-winlog_x_winlog: *indexSettings
-    so-logs-apache_x_access: *indexSettings
-    so-logs-apache_x_error: *indexSettings
-    so-logs-auditd_x_log: *indexSettings
-    so-logs-aws_x_cloudtrail: *indexSettings
-    so-logs-aws_x_cloudwatch_logs: *indexSettings
-    so-logs-aws_x_ec2_logs: *indexSettings
-    so-logs-aws_x_elb_logs: *indexSettings
-    so-logs-aws_x_firewall_logs: *indexSettings
-    so-logs-aws_x_route53_public_logs: *indexSettings
-    so-logs-aws_x_route53_resolver_logs: *indexSettings
-    so-logs-aws_x_s3access: *indexSettings
-    so-logs-aws_x_vpcflow: *indexSettings
-    so-logs-aws_x_waf: *indexSettings
-    so-logs-azure_x_activitylogs: *indexSettings
-    so-logs-azure_x_application_gateway: *indexSettings
-    so-logs-azure_x_auditlogs: *indexSettings
-    so-logs-azure_x_eventhub: *indexSettings
-    so-logs-azure_x_firewall_logs: *indexSettings
-    so-logs-azure_x_identity_protection: *indexSettings
-    so-logs-azure_x_platformlogs: *indexSettings
-    so-logs-azure_x_provisioning: *indexSettings
-    so-logs-azure_x_signinlogs: *indexSettings
-    so-logs-azure_x_springcloudlogs: *indexSettings
-    so-logs-barracuda_x_waf: *indexSettings
-    so-logs-barracuda_cloudgen_firewall_x_log: *indexSettings
-    so-logs-cef_x_log: *indexSettings
-    so-logs-cisco_asa_x_log: *indexSettings
-    so-logs-cisco_ftd_x_log: *indexSettings
-    so-logs-cisco_ios_x_log: *indexSettings
-    so-logs-cisco_ise_x_log: *indexSettings
-    so-logs-citrix_adc_x_interface: *indexSettings
-    so-logs-citrix_adc_x_lbvserver: *indexSettings
-    so-logs-citrix_adc_x_service: *indexSettings
-    so-logs-citrix_adc_x_system: *indexSettings
-    so-logs-citrix_adc_x_vpn: *indexSettings
-    so-logs-citrix_waf_x_log: *indexSettings
-    so-logs-cloudflare_x_audit: *indexSettings
-    so-logs-cloudflare_x_logpull: *indexSettings
-    so-logs-crowdstrike_x_falcon: *indexSettings
-    so-logs-crowdstrike_x_fdr: *indexSettings
-    so-logs-darktrace_x_ai_analyst_alert: *indexSettings
-    so-logs-darktrace_x_model_breach_alert: *indexSettings
-    so-logs-darktrace_x_system_status_alert: *indexSettings
     so-logs-detections_x_alerts: *indexSettings
-    so-logs-f5_bigip_x_log: *indexSettings
-    so-logs-fim_x_event: *indexSettings
-    so-logs-fortinet_x_clientendpoint: *indexSettings
-    so-logs-fortinet_x_firewall: *indexSettings
-    so-logs-fortinet_x_fortimail: *indexSettings
-    so-logs-fortinet_x_fortimanager: *indexSettings
-    so-logs-fortinet_x_fortigate: *indexSettings
-    so-logs-gcp_x_audit: *indexSettings
-    so-logs-gcp_x_dns: *indexSettings
-    so-logs-gcp_x_firewall: *indexSettings
-    so-logs-gcp_x_loadbalancing_logs: *indexSettings
-    so-logs-gcp_x_vpcflow: *indexSettings
-    so-logs-github_x_audit: *indexSettings
-    so-logs-github_x_code_scanning: *indexSettings
-    so-logs-github_x_dependabot: *indexSettings
-    so-logs-github_x_issues: *indexSettings
-    so-logs-github_x_secret_scanning: *indexSettings
-    so-logs-google_workspace_x_access_transparency: *indexSettings
-    so-logs-google_workspace_x_admin: *indexSettings
-    so-logs-google_workspace_x_alert: *indexSettings
-    so-logs-google_workspace_x_context_aware_access: *indexSettings
-    so-logs-google_workspace_x_device: *indexSettings
-    so-logs-google_workspace_x_drive: *indexSettings
-    so-logs-google_workspace_x_gcp: *indexSettings
-    so-logs-google_workspace_x_group_enterprise: *indexSettings
-    so-logs-google_workspace_x_groups: *indexSettings
-    so-logs-google_workspace_x_login: *indexSettings
-    so-logs-google_workspace_x_rules: *indexSettings
-    so-logs-google_workspace_x_saml: *indexSettings
-    so-logs-google_workspace_x_token: *indexSettings
-    so-logs-google_workspace_x_user_accounts: *indexSettings
     so-logs-http_endpoint_x_generic: *indexSettings
     so-logs-httpjson_x_generic: *indexSettings
-    so-logs-iis_x_access: *indexSettings
-    so-logs-iis_x_error: *indexSettings
-    so-logs-imperva_cloud_waf_x_event: *indexSettings
-    so-logs-juniper_x_junos: *indexSettings
-    so-logs-juniper_x_netscreen: *indexSettings
-    so-logs-juniper_x_srx: *indexSettings
-    so-logs-juniper_srx_x_log: *indexSettings
-    so-logs-kafka_log_x_generic: *indexSettings
-    so-logs-lastpass_x_detailed_shared_folder: *indexSettings
-    so-logs-lastpass_x_event_report: *indexSettings
-    so-logs-lastpass_x_user: *indexSettings
-    so-logs-m365_defender_x_event: *indexSettings
-    so-logs-m365_defender_x_incident: *indexSettings
-    so-logs-m365_defender_x_log: *indexSettings
-    so-logs-microsoft_defender_endpoint_x_log: *indexSettings
-    so-logs-microsoft_dhcp_x_log: *indexSettings
-    so-logs-microsoft_sqlserver_x_audit: *indexSettings
-    so-logs-microsoft_sqlserver_x_log: *indexSettings
-    so-logs-mysql_x_error: *indexSettings
-    so-logs-mysql_x_slowlog: *indexSettings
-    so-logs-netflow_x_log: *indexSettings
-    so-logs-nginx_x_access: *indexSettings
-    so-logs-nginx_x_error: *indexSettings
-    so-logs-o365_x_audit: *indexSettings
-    so-logs-okta_x_system: *indexSettings
-    so-logs-panw_x_panos: *indexSettings
-    so-logs-pfsense_x_log: *indexSettings
-    so-logs-proofpoint_tap_x_clicks_blocked: *indexSettings
-    so-logs-proofpoint_tap_x_clicks_permitted: *indexSettings
-    so-logs-proofpoint_tap_x_message_blocked: *indexSettings
-    so-logs-proofpoint_tap_x_message_delivered: *indexSettings
-    so-logs-sentinel_one_x_activity: *indexSettings
-    so-logs-sentinel_one_x_agent: *indexSettings
-    so-logs-sentinel_one_x_alert: *indexSettings
-    so-logs-sentinel_one_x_group: *indexSettings
-    so-logs-sentinel_one_x_threat: *indexSettings
-    so-logs-sonicwall_firewall_x_log: *indexSettings
-    so-logs-snort_x_log: *indexSettings
-    so-logs-symantec_endpoint_x_log: *indexSettings
-    so-logs-tenable_io_x_asset: *indexSettings
-    so-logs-tenable_io_x_plugin: *indexSettings
-    so-logs-tenable_io_x_scan: *indexSettings
-    so-logs-tenable_io_x_vulnerability: *indexSettings
-    so-logs-tenable_sc_x_asset: *indexSettings
-    so-logs-tenable_sc_x_plugin: *indexSettings
-    so-logs-tenable_sc_x_vulnerability: *indexSettings
-    so-logs-ti_abusech_x_malware: *indexSettings
-    so-logs-ti_abusech_x_malwarebazaar: *indexSettings
-    so-logs-ti_abusech_x_threatfox: *indexSettings
-    so-logs-ti_abusech_x_url: *indexSettings
-    so-logs-ti_anomali_x_threatstream: *indexSettings
-    so-logs-ti_cybersixgill_x_threat: *indexSettings
-    so-logs-ti_misp_x_threat: *indexSettings
-    so-logs-ti_misp_x_threat_attributes: *indexSettings
-    so-logs-ti_otx_x_pulses_subscribed: *indexSettings  
-    so-logs-ti_otx_x_threat: *indexSettings
-    so-logs-ti_recordedfuture_x_latest_ioc-template: *indexSettings
-    so-logs-ti_recordedfuture_x_threat: *indexSettings
-    so-logs-ti_threatq_x_threat: *indexSettings
-    so-logs-zscaler_zia_x_alerts: *indexSettings
-    so-logs-zscaler_zia_x_dns: *indexSettings
-    so-logs-zscaler_zia_x_firewall: *indexSettings
-    so-logs-zscaler_zia_x_tunnel: *indexSettings
-    so-logs-zscaler_zia_x_web: *indexSettings
-    so-logs-zscaler_zpa_x_app_connector_status: *indexSettings
-    so-logs-zscaler_zpa_x_audit: *indexSettings
-    so-logs-zscaler_zpa_x_browser_access: *indexSettings
-    so-logs-zscaler_zpa_x_user_activity: *indexSettings
-    so-logs-zscaler_zpa_x_user_status: *indexSettings
-    so-logs-1password_x_item_usages: *indexSettings
-    so-logs-1password_x_signin_attempts: *indexSettings
     so-logs-osquery-manager-actions: *indexSettings
     so-logs-osquery-manager-action_x_responses: *indexSettings
+    so-logs-osquery-manager_x_action_x_responses: *indexSettings
+    so-logs-osquery-manager_x_result: *indexSettings
     so-logs-elastic_agent_x_apm_server: *indexSettings
     so-logs-elastic_agent_x_auditbeat: *indexSettings
     so-logs-elastic_agent_x_cloudbeat: *indexSettings
@@ -531,6 +396,9 @@ elasticsearch:
     so-metrics-endpoint_x_metrics: *indexSettings
     so-metrics-endpoint_x_policy: *indexSettings
     so-metrics-nginx_x_stubstatus: *indexSettings
+    so-metrics-vsphere_x_datastore: *indexSettings
+    so-metrics-vsphere_x_host: *indexSettings
+    so-metrics-vsphere_x_virtualmachine: *indexSettings
     so-case: *indexSettings
     so-common: *indexSettings
     so-endgame: *indexSettings
@@ -539,6 +407,7 @@ elasticsearch:
     so-suricata_x_alerts: *indexSettings
     so-import: *indexSettings
     so-kratos: *indexSettings
+    so-hydra: *indexSettings
     so-kismet: *indexSettings
     so-logstash: *indexSettings
     so-redis: *indexSettings

salt/elasticsearch/template.map.jinja

@@ -14,6 +14,18 @@
 
 {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
 
+{# start generation of integration default index_settings #}
+{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') and salt['file.file_exists']('/opt/so/state/esfleet_component_templates.json') %}
+{%   set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
+{%   if check_package_components.size > 1 %}
+{%    from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
+{%    for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %}
+{%      do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %}
+{%    endfor %}
+{%   endif%}
+{% endif %}
+{# end generation of integration default index_settings #}
+
 {% set ES_INDEX_SETTINGS_GLOBAL_OVERRIDES = {} %}
 {% for index in ES_INDEX_SETTINGS_ORIG.keys() %}
 {%   do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}

salt/elasticsearch/templates/component/ecs/hash.json

@@ -0,0 +1,69 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "hash": {
+          "type": "object",
+          "properties": {
+            "ja3": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja3s": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "hassh": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "md5": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "sha1": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "sha256": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4l": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4ls": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4t": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4ts": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4ssh": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4h": {
+              "type": "keyword",
+              "ignore_above": 1024
+            },
+            "ja4x": {
+              "type": "keyword",
+              "ignore_above": 1024
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/log.json

@@ -29,7 +29,7 @@
                 "file": {
                   "properties": {
                     "line": {
-                      "type": "integer"
+                      "type": "long"
                     },
                     "name": {
                       "ignore_above": 1024,

salt/elasticsearch/templates/component/ecs/metadata.json

@@ -0,0 +1,26 @@
+{
+  "template": {
+    "mappings": {
+      "dynamic_templates": [],
+      "properties": {
+        "metadata": {
+          "properties": {
+            "kafka": {
+              "properties": {
+                "timestamp": {
+                  "type": "date"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "_meta": {
+    "_meta": {
+      "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-log.html",
+      "ecs_version": "1.12.2"
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/zeek.json

@@ -603,6 +603,89 @@
                 }
               }
             },
+            "ipsec": {
+              "properties": {
+                "certificates": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "exchange_type": {
+                  "type": "short"
+                },
+                "flag_a": {
+                  "type": "boolean"
+                },
+                "flag_c": {
+                  "type": "boolean"
+                },
+                "flag_e": {
+                  "type": "boolean"
+                },
+                "flag_i": {
+                  "type": "boolean"
+                },
+                "flag_r": {
+                  "type": "boolean"
+                },
+                "flag_v": {
+                  "type": "boolean"
+                },
+                "hash": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "initiator_spi": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "ke_dh_groups": {
+                  "type": "short"
+                },
+                "length": {
+                  "type": "long"
+                },
+                "maj_version": {
+                  "type": "short"
+                },
+                "message_id": {
+                  "type": "long"
+                },
+                "min_version": {
+                  "type": "short"
+                },
+                "notify_messages": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "proposals": {
+                  "type": "long"
+                },
+                "responder_spi": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "situation": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "transform_attributes": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "transforms": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "vendor_ids": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "version": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            },
             "irc": {
               "properties": {
                 "addl": {
@@ -751,6 +834,81 @@
                 }
               }
             },
+            "ldap": {
+              "type": "object",
+              "properties": {
+                "message_id": {
+                  "type": "short"
+                },
+                "opcode": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "result": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "diagnostic_message": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "version": {
+                  "type": "short"
+                },
+                "object": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "argument": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "user_email": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "property": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "common_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "organizational_unit": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "domain": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            },
+            "ldap_search": {
+              "type": "object",
+              "properties": {
+                "scope": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "deref_aliases": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "result_count": {
+                  "type": "long"
+                },
+                "filter": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "attributes": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            },
             "modbus": {
               "properties": {
                 "exception": {
@@ -1089,6 +1247,38 @@
                 }
               }
             },
+            "quic": {
+              "type": "object",
+              "properties": {
+                "server_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "version": {
+                  "type": "short"
+                },
+                "client_initial_dcid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "client_scid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "server_scid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "client_protocol": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "history": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            },
             "radius": {
               "properties": {
                 "connect_info": {

salt/elasticsearch/templates/component/elastic-agent/logs-1password.item_usages@custom.json

@@ -1,36 +0,0 @@
-{
-  "template": {
-    "mappings": {
-      "properties": {
-        "host": {
-	  "properties":{
-            "ip": {
-              "type": "ip"
-            }
-	  }
-	},
-	"related": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"destination": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"source": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/logs-1password.signin_attempts@custom.json

@@ -1,36 +0,0 @@
-{
-  "template": {
-    "mappings": {
-      "properties": {
-        "host": {
-	  "properties":{
-            "ip": {
-              "type": "ip"
-            }
-	  }
-	},
-	"related": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"destination": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"source": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/logs-apache.access@custom.json

@@ -1,36 +0,0 @@
-{
-  "template": {
-    "mappings": {
-      "properties": {
-        "host": {
-	  "properties":{
-            "ip": {
-              "type": "ip"
-            }
-	  }
-	},
-	"related": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"destination": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"source": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/logs-apache.error@custom.json

@@ -1,36 +0,0 @@
-{
-  "template": {
-    "mappings": {
-      "properties": {
-        "host": {
-	  "properties":{
-            "ip": {
-              "type": "ip"
-            }
-	  }
-	},
-	"related": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"destination": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"source": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/logs-auditd.log@custom.json

@@ -1,36 +0,0 @@
-{
-  "template": {
-    "mappings": {
-      "properties": {
-        "host": {
-	  "properties":{
-            "ip": {
-              "type": "ip"
-            }
-	  }
-	},
-	"related": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"destination": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"source": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/logs-auth0.logs@custom.json

@@ -1,36 +0,0 @@
-{
-  "template": {
-    "mappings": {
-      "properties": {
-        "host": {
-	  "properties":{
-            "ip": {
-              "type": "ip"
-            }
-	  }
-	},
-	"related": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"destination": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"source": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/logs-aws.cloudfront_logs@custom.json

@@ -1,36 +0,0 @@
-{
-  "template": {
-    "mappings": {
-      "properties": {
-        "host": {
-	  "properties":{
-            "ip": {
-              "type": "ip"
-            }
-	  }
-	},
-	"related": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"destination": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"source": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/logs-aws.cloudtrail@custom.json

@@ -1,36 +0,0 @@
-{
-  "template": {
-    "mappings": {
-      "properties": {
-        "host": {
-	  "properties":{
-            "ip": {
-              "type": "ip"
-            }
-	  }
-	},
-	"related": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"destination": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"source": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/logs-aws.cloudwatch_logs@custom.json

@@ -1,36 +0,0 @@
-{
-  "template": {
-    "mappings": {
-      "properties": {
-        "host": {
-	  "properties":{
-            "ip": {
-              "type": "ip"
-            }
-	  }
-	},
-	"related": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"destination": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"source": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        }
-      }
-    }
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/logs-aws.ec2_logs@custom.json

@@ -1,36 +0,0 @@
-{
-  "template": {
-    "mappings": {
-      "properties": {
-        "host": {
-	  "properties":{
-            "ip": {
-              "type": "ip"
-            }
-	  }
-	},
-	"related": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"destination": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        },
-	"source": {
-          "properties":{
-            "ip": {
-              "type": "ip"
-            }
-          }
-        }
-      }
-    }
-  }
-}