Merge pull request #14612 from Security-Onion-Solutions/2.4/dev

2.4.150

.github/.gitleaks.toml

@@ -536,7 +536,7 @@ secretGroup = 4
 
 [allowlist]
 description = "global allow lists"
-regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''', '''ssl_.*password''']
+regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''', '''ssl_.*password''', '''integration_key\s=\s"so-logs-"''']
 paths = [
     '''gitleaks.toml''',
     '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -11,7 +11,6 @@ body:
       description: Which version of Security Onion 2.4.x are you asking about?
       options:
         -
-        - 2.4 Pre-release (Beta, Release Candidate)
         - 2.4.10
         - 2.4.20
         - 2.4.30
@@ -22,6 +21,13 @@ body:
         - 2.4.80
         - 2.4.90
         - 2.4.100
+        - 2.4.110
+        - 2.4.111
+        - 2.4.120
+        - 2.4.130
+        - 2.4.140
+        - 2.4.141
+        - 2.4.150
         - Other (please provide detail below)
     validations:
       required: true
@@ -32,9 +38,10 @@ body:
       options:
         -
         - Security Onion ISO image
-        - Network installation on Red Hat derivative like Oracle, Rocky, Alma, etc.
-        - Network installation on Ubuntu
-        - Network installation on Debian
+        - Cloud image (Amazon, Azure, Google)
+        - Network installation on Red Hat derivative like Oracle, Rocky, Alma, etc. (unsupported)
+        - Network installation on Ubuntu (unsupported)
+        - Network installation on Debian (unsupported)
         - Other (please provide detail below)
     validations:
       required: true

.github/ISSUE_TEMPLATE

@@ -1,12 +0,0 @@
-PLEASE STOP AND READ THIS INFORMATION!
-
-If you are creating an issue just to ask a question, you will likely get faster and better responses by posting to our discussions forum instead:
-https://securityonion.net/discuss
-
-If you think you have found a possible bug or are observing a behavior that you weren't expecting, use the discussion forum to start a conversation about it instead of creating an issue.
-
-If you are very familiar with the latest version of the product and are confident you have found a bug in Security Onion, you can continue with creating an issue here, but please make sure you have done the following:
-- duplicated the issue on a fresh installation of the latest version
-- provide information about your system and how you installed Security Onion
-- include relevant log files
-- include reproduction steps

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,38 @@
+---
+name: Bug report
+about: This option is for experienced community members to report a confirmed, reproducible bug
+title: ''
+labels: ''
+assignees: ''
+
+---
+PLEASE STOP AND READ THIS INFORMATION!
+
+If you are creating an issue just to ask a question, you will likely get faster and better responses by posting to our discussions forum at https://securityonion.net/discuss.
+
+If you think you have found a possible bug or are observing a behavior that you weren't expecting, use the discussion forum at https://securityonion.net/discuss to start a conversation about it instead of creating an issue.
+
+If you are very familiar with the latest version of the product and are confident you have found a bug in Security Onion, you can continue with creating an issue here, but please make sure you have done the following:
+- duplicated the issue on a fresh installation of the latest version
+- provide information about your system and how you installed Security Onion
+- include relevant log files
+- include reproduction steps
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Security Onion Discussions
+    url: https://securityonion.com/discussions
+    about: Please ask and answer questions here

.github/workflows/contrib.yml

@@ -18,7 +18,7 @@ jobs:
         with:
           path-to-signatures: 'signatures_v1.json'
           path-to-document: 'https://securityonionsolutions.com/cla' 
-          allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,weslambert,defensivedepth,m0duspwnens
+          allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,defensivedepth,m0duspwnens
           remote-organization-name: Security-Onion-Solutions
           remote-repository-name: licensing
 

.github/workflows/pythontest.yml

@@ -1,10 +1,6 @@
 name: python-test
 
 on:
-  push:
-    paths: 
-      - "salt/sensoroni/files/analyzers/**"
-      - "salt/manager/tools/sbin"
   pull_request:
     paths:
       - "salt/sensoroni/files/analyzers/**"
@@ -17,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.13"]
         python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"]
 
     steps:
@@ -36,4 +32,4 @@ jobs:
         flake8 ${{ matrix.python-code-path }} --show-source --max-complexity=12  --doctests --max-line-length=200 --statistics
     - name: Test with pytest
       run: |
-        pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini
+        PYTHONPATH=${{ matrix.python-code-path }} pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.100-20240903 ISO image released on 2024/09/03
+### 2.4.150-20250512 ISO image released on 2025/05/12
 
 
 ### Download and Verify
 
-2.4.100-20240903 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.100-20240903.iso
+2.4.150-20250512 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.150-20250512.iso
  
-MD5: 856BBB4F0764C0A479D8949725FC096B  
-SHA1: B3FCFB8F1031EB8AA833A90C6C5BB61328A73842  
-SHA256: 0103EB9D78970396BB47CBD18DA1FFE64524F5C1C559487A1B2D293E1882B265  
+MD5: 7A7469A7A38EA9A2DB770C36AE36A0CA  
+SHA1: 7E768D515353F339DC536DED6207B786DAFF7D27  
+SHA256: F8B2EB6B332F2367F0C097D211577565C8FB5CC7809E97D63687C634035B3699  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.100-20240903.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.150-20250512.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.100-20240903.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.150-20250512.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.100-20240903.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.150-20250512.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.100-20240903.iso.sig securityonion-2.4.100-20240903.iso
+gpg --verify securityonion-2.4.150-20250512.iso.sig securityonion-2.4.150-20250512.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Sat 31 Aug 2024 05:05:05 PM EDT using RSA key ID FE507013
+gpg: Signature made Fri 09 May 2025 06:27:29 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

HOTFIX

@@ -1 +0,0 @@
-20240903

LICENSE

@@ -0,0 +1,53 @@
+Elastic License 2.0 (ELv2)
+
+Acceptance
+
+  By using the software, you agree to all of the terms and conditions below.
+
+Copyright License
+
+  The licensor grants you a non-exclusive, royalty-free, worldwide, non-sublicensable, non-transferable license to use, copy, distribute, make available, and prepare derivative works of the software, in each case subject to the limitations and conditions below.
+
+Limitations
+
+  You may not provide the software to third parties as a hosted or managed service, where the service provides users with access to any substantial set of the features or functionality of the software.
+
+  You may not move, change, disable, or circumvent the license key functionality in the software, and you may not remove or obscure any functionality in the software that is protected by the license key.
+
+  You may not alter, remove, or obscure any licensing, copyright, or other notices of the licensor in the software. Any use of the licensor’s trademarks is subject to applicable law.
+
+Patents
+
+  The licensor grants you a license, under any patent claims the licensor can license, or becomes able to license, to make, have made, use, sell, offer for sale, import and have imported the software, in each case subject to the limitations and conditions in this license. This license does not cover any patent claims that you cause to be infringed by modifications or additions to the software. If you or your company make any written claim that the software infringes or contributes to infringement of any patent, your patent license for the software granted under these terms ends immediately. If your company makes such a claim, your patent license ends immediately for work on behalf of your company.
+
+Notices
+
+  You must ensure that anyone who gets a copy of any part of the software from you also gets a copy of these terms.
+
+  If you modify the software, you must include in any modified copies of the software prominent notices stating that you have modified the software.
+
+No Other Rights
+
+  These terms do not imply any licenses other than those expressly granted in these terms.
+
+Termination
+
+  If you use the software in violation of these terms, such use is not licensed, and your licenses will automatically terminate. If the licensor provides you with a notice of your violation, and you cease all violation of this license no later than 30 days after you receive that notice, your licenses will be reinstated retroactively. However, if you violate these terms after such reinstatement, any additional violation of these terms will cause your licenses to terminate automatically and permanently.
+
+No Liability
+
+  As far as the law allows, the software comes as is, without any warranty or condition, and the licensor will not be liable to you for any damages arising out of these terms or the use or nature of the software, under any kind of legal claim.
+
+Definitions
+
+  The licensor is the entity offering these terms, and the software is the software the licensor makes available under these terms, including any portion of it.
+
+  you refers to the individual or entity agreeing to these terms.
+
+  your company is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization. control means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise. Control can be direct or indirect.
+
+  your licenses are all the licenses granted to you for the software under these terms.
+
+  use means anything you do with the software requiring one of your licenses.
+  
+  trademark means trademarks, service marks, and similar rights.

VERSION

@@ -1 +1 @@
-2.4.100
+2.4.150

pillar/node_data/ips.sls

@@ -24,6 +24,7 @@
 {%   endif %}
 {% endfor %}
 
+{% if node_types %}
 node_data:
 {% for node_type, host_values in node_types.items() %}
 {%   for hostname, details in host_values.items() %}
@@ -33,3 +34,6 @@ node_data:
     role: {{node_type}}
 {%   endfor %}
 {% endfor %}
+{% else %}
+node_data: False
+{% endif %}

pillar/top.sls

@@ -16,16 +16,18 @@ base:
     - sensoroni.adv_sensoroni
     - telegraf.soc_telegraf
     - telegraf.adv_telegraf
+    - versionlock.soc_versionlock
+    - versionlock.adv_versionlock
 
   '* and not *_desktop':
     - firewall.soc_firewall
     - firewall.adv_firewall
     - nginx.soc_nginx
     - nginx.adv_nginx
-    - node_data.ips
 
   '*_manager or *_managersearch':
     - match: compound
+    - node_data.ips
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
@@ -47,6 +49,8 @@ base:
     - kibana.adv_kibana
     - kratos.soc_kratos
     - kratos.adv_kratos
+    - hydra.soc_hydra
+    - hydra.adv_hydra
     - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
@@ -86,6 +90,7 @@ base:
     - soc.license
 
   '*_eval':
+    - node_data.ips
     - secrets
     - healthcheck.eval
     - elasticsearch.index_templates
@@ -96,6 +101,7 @@ base:
     - kibana.secrets
     {% endif %}
     - kratos.soc_kratos
+    - kratos.adv_kratos
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     - elasticfleet.soc_elasticfleet
@@ -113,8 +119,8 @@ base:
     - kibana.adv_kibana
     - strelka.soc_strelka
     - strelka.adv_strelka
-    - kratos.soc_kratos
-    - kratos.adv_kratos
+    - hydra.soc_hydra
+    - hydra.adv_hydra
     - redis.soc_redis
     - redis.adv_redis
     - influxdb.soc_influxdb
@@ -133,6 +139,7 @@ base:
     - minions.adv_{{ grains.id }}
 
   '*_standalone':
+    - node_data.ips
     - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
@@ -149,6 +156,8 @@ base:
     - idstools.adv_idstools
     - kratos.soc_kratos
     - kratos.adv_kratos
+    - hydra.soc_hydra
+    - hydra.adv_hydra
     - redis.nodes
     - redis.soc_redis
     - redis.adv_redis
@@ -253,6 +262,7 @@ base:
     - soc.license
 
   '*_import':
+    - node_data.ips
     - secrets
     - elasticsearch.index_templates
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
@@ -262,6 +272,7 @@ base:
     - kibana.secrets
     {% endif %}
     - kratos.soc_kratos
+    - kratos.adv_kratos
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     - elasticfleet.soc_elasticfleet
@@ -277,8 +288,8 @@ base:
     - kibana.adv_kibana
     - backup.soc_backup
     - backup.adv_backup
-    - kratos.soc_kratos
-    - kratos.adv_kratos
+    - hydra.soc_hydra
+    - hydra.adv_hydra
     - redis.soc_redis
     - redis.adv_redis
     - influxdb.soc_influxdb
@@ -297,6 +308,7 @@ base:
     - minions.adv_{{ grains.id }}
 
   '*_fleet':
+    - node_data.ips
     - backup.soc_backup
     - backup.adv_backup
     - logstash.nodes
@@ -310,3 +322,5 @@ base:
   '*_desktop':
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - stig.soc_stig
+    - soc.license

salt/allowed_states.map.jinja

@@ -24,6 +24,7 @@
           'influxdb',
           'soc',
           'kratos',
+          'hydra',
           'elasticfleet',
           'elastic-fleet-package-registry',
           'firewall',
@@ -68,6 +69,7 @@
           'strelka.manager',
           'soc',
           'kratos',
+          'hydra',
           'influxdb',
           'telegraf',
           'firewall',
@@ -95,6 +97,7 @@
           'strelka.manager',
           'soc',
           'kratos',
+          'hydra',
           'elasticfleet',
           'elastic-fleet-package-registry',
           'firewall',
@@ -117,6 +120,7 @@
           'strelka.manager',
           'soc',
           'kratos',
+          'hydra',
           'elastic-fleet-package-registry',
           'elasticfleet',
           'firewall',
@@ -151,6 +155,7 @@
           'influxdb',
           'soc',
           'kratos',
+          'hydra',
           'elastic-fleet-package-registry',
           'elasticfleet',
           'firewall',
@@ -202,7 +207,8 @@
       'so-desktop': [
           'ssl',
           'docker_clean',
-          'telegraf'
+          'telegraf',
+          'stig'
           ],
   }, grain='role') %}
 

salt/backup/defaults.yaml

@@ -4,4 +4,5 @@ backup:
     - /etc/pki
     - /etc/salt
     - /nsm/kratos
+    - /nsm/hydra
   destination: "/nsm/backup"

salt/common/init.sls

@@ -128,6 +128,11 @@ common_sbin:
     - user: 939
     - group: 939
     - file_mode: 755
+    - show_changes: False
+{% if GLOBALS.role == 'so-heavynode' %}
+    - exclude_pat:
+      - so-pcap-import
+{% endif %}
 
 common_sbin_jinja:
   file.recurse:
@@ -137,6 +142,21 @@ common_sbin_jinja:
     - group: 939 
     - file_mode: 755
     - template: jinja
+    - show_changes: False
+{% if GLOBALS.role == 'so-heavynode' %}
+    - exclude_pat:
+      - so-import-pcap
+{% endif %}
+
+{% if GLOBALS.role == 'so-heavynode' %}
+remove_so-pcap-import_heavynode:
+  file.absent:
+    - name: /usr/sbin/so-pcap-import
+
+remove_so-import-pcap_heavynode:
+  file.absent:
+    - name: /usr/sbin/so-import-pcap
+{% endif %}
 
 {% if not GLOBALS.is_manager%}
 # prior to 2.4.50 these scripts were in common/tools/sbin on the manager because of soup and distributed to non managers
@@ -182,6 +202,7 @@ sostatus_log:
   file.managed:
     - name: /opt/so/log/sostatus/status.log
     - mode: 644
+    - replace: False
 
 # Install sostatus check cron. This is used to populate Grid.
 so-status_check_cron:

salt/common/packages.sls

@@ -27,6 +27,7 @@ commonpkgs:
       - vim
       - tar
       - unzip
+      - bc
       {% if grains.oscodename != 'focal' %}
       - python3-rich
       {% endif %}
@@ -56,6 +57,7 @@ commonpkgs:
     - skip_suggestions: True
     - pkgs:
       - python3-dnf-plugin-versionlock
+      - bc
       - curl
       - device-mapper-persistent-data
       - fuse

salt/common/soup_scripts.sls

@@ -11,6 +11,7 @@
 {%   else %}
 {%     set UPDATE_DIR='/tmp/sogh/securityonion' %}
 {%   endif %}
+{%   set SOVERSION = salt['file.read']('/etc/soversion').strip() %}
 
 remove_common_soup:
   file.absent:
@@ -63,6 +64,12 @@ copy_so-repo-sync_manager_tools_sbin:
     - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
     - preserve: True
 
+copy_bootstrap-salt_manager_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/salt/scripts/bootstrap-salt.sh
+    - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
+    - preserve: True
+
 # This section is used to put the new script in place so that it can be called during soup.
 # It is faster than calling the states that normally manage them to put them in place.
 copy_so-common_sbin:
@@ -107,6 +114,24 @@ copy_so-repo-sync_sbin:
     - force: True
     - preserve: True
 
+copy_bootstrap-salt_sbin:
+  file.copy:
+    - name: /usr/sbin/bootstrap-salt.sh
+    - source: {{UPDATE_DIR}}/salt/salt/scripts/bootstrap-salt.sh
+    - force: True
+    - preserve: True
+
+{# this is added in 2.4.120 to remove salt repo files pointing to saltproject.io to accomodate the move to broadcom and new bootstrap-salt script #}
+{%   if salt['pkg.version_cmp'](SOVERSION, '2.4.120') == -1 %}
+{%     set saltrepofile = '/etc/yum.repos.d/salt.repo' %}
+{%     if grains.os_family == 'Debian' %}
+{%       set saltrepofile = '/etc/apt/sources.list.d/salt.list' %}
+{%     endif %}
+remove_saltproject_io_repo_manager:
+  file.absent:
+    - name: {{ saltrepofile }}
+{%   endif %}
+
 {% else %}
 fix_23_soup_sbin:
   cmd.run:

salt/common/tools/sbin/so-common

@@ -8,12 +8,6 @@
 # Elastic agent is not managed by salt. Because of this we must store this base information in a 
 # script that accompanies the soup system. Since so-common is one of those special soup files, 
 # and since this same logic is required during installation, it's included in this file.
-ELASTIC_AGENT_TARBALL_VERSION="8.14.3"
-ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
-ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
-ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
-ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
-ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
 
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
@@ -174,6 +168,46 @@ check_salt_minion_status() {
 	return $status
 }
 
+# Compare es versions and return the highest version
+compare_es_versions() {
+    # Save the original IFS
+    local OLD_IFS="$IFS"
+
+    IFS=.
+    local i ver1=($1) ver2=($2)
+
+    # Restore the original IFS
+    IFS="$OLD_IFS"
+
+    # Determine the maximum length between the two version arrays
+    local max_len=${#ver1[@]}
+    if [[ ${#ver2[@]} -gt $max_len ]]; then
+        max_len=${#ver2[@]}
+    fi
+
+    # Compare each segment of the versions
+    for ((i=0; i<max_len; i++)); do
+		# If a segment in ver1 or ver2 is missing, set it to 0
+        if [[ -z ${ver1[i]} ]]; then
+            ver1[i]=0
+        fi
+        if [[ -z ${ver2[i]} ]]; then
+            ver2[i]=0
+        fi
+        if ((10#${ver1[i]} > 10#${ver2[i]})); then
+            echo "$1"
+            return 0
+        fi
+        if ((10#${ver1[i]} < 10#${ver2[i]})); then
+            echo "$2"
+            return 0
+        fi
+    done
+
+    echo "$1"  # If versions are equal, return either
+    return 0
+}
+
 copy_new_files() {
   # Copy new files over to the salt dir
   cd $UPDATE_DIR
@@ -192,7 +226,7 @@ create_local_directories() {
 		for d in $(find $PILLARSALTDIR/$i -type d); do
 			suffixdir=${d//$PILLARSALTDIR/}
 			if [ ! -d "$local_salt_dir/$suffixdir" ]; then
-				mkdir -pv $local_salt_dir$suffixdir
+				mkdir -p $local_salt_dir$suffixdir
 			fi
 		done
 		chown -R socore:socore $local_salt_dir/$i
@@ -263,11 +297,6 @@ fail() {
 	exit 1
 }
 
-get_random_value() {
-	length=${1:-20}
-	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
-}
-
 get_agent_count() {
 	if [ -f /opt/so/log/agents/agentstatus.log ]; then
 		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}')
@@ -276,6 +305,27 @@ get_agent_count() {
 	fi
 }
 
+get_elastic_agent_vars() {
+	local path="${1:-/opt/so/saltstack/default}"
+	local defaultsfile="${path}/salt/elasticsearch/defaults.yaml"
+
+	if [ -f "$defaultsfile" ]; then
+		ELASTIC_AGENT_TARBALL_VERSION=$(egrep " +version: " $defaultsfile | awk -F: '{print $2}' | tr -d '[:space:]')
+		ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+		ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+		ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+		ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+		ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
+	else
+		fail "Could not find salt/elasticsearch/defaults.yaml"
+	fi
+}
+
+get_random_value() {
+	length=${1:-20}
+	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
+}
+
 gpg_rpm_import() {
 	if [[ $is_oracle ]]; then
 	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
@@ -627,6 +677,8 @@ has_uppercase() {
 }
 
 update_elastic_agent() {
+	local path="${1:-/opt/so/saltstack/default}"
+	get_elastic_agent_vars "$path"
 	echo "Checking if Elastic Agent update is necessary..."
 	download_and_verify "$ELASTIC_AGENT_URL" "$ELASTIC_AGENT_MD5_URL" "$ELASTIC_AGENT_FILE" "$ELASTIC_AGENT_MD5" "$ELASTIC_AGENT_EXPANSION_DIR"
 }

salt/common/tools/sbin/so-image-common

@@ -29,6 +29,7 @@ container_list() {
       "so-influxdb"
       "so-kibana"
       "so-kratos"
+      "so-hydra"
       "so-nginx"
       "so-pcaptools"
       "so-soc"
@@ -53,6 +54,7 @@ container_list() {
       "so-kafka"
       "so-kibana"
       "so-kratos"
+      "so-hydra"
       "so-logstash"
       "so-nginx"
       "so-pcaptools"
@@ -112,6 +114,10 @@ update_docker_containers() {
     container_list
   fi
 
+  # all the images using ELASTICSEARCHDEFAULTS.elasticsearch.version
+  # does not include so-elastic-fleet since that container uses so-elastic-agent image
+  local IMAGES_USING_ES_VERSION=("so-elasticsearch")
+
   rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1 
   mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 
 
@@ -139,15 +145,36 @@ update_docker_containers() {
       $PROGRESS_CALLBACK $i
     fi
 
+    if [[ " ${IMAGES_USING_ES_VERSION[*]} " =~ [[:space:]]${i}[[:space:]] ]]; then
+      # this is an es container so use version defined in elasticsearch defaults.yaml
+      local UPDATE_DIR='/tmp/sogh/securityonion'
+      if [ ! -d "$UPDATE_DIR" ]; then
+        UPDATE_DIR=/securityonion
+      fi
+      local v1=0
+      local v2=0
+      if [[ -f "$UPDATE_DIR/salt/elasticsearch/defaults.yaml" ]]; then
+        v1=$(egrep " +version: " "$UPDATE_DIR/salt/elasticsearch/defaults.yaml" | awk -F: '{print $2}' | tr -d '[:space:]')
+      fi
+      if [[ -f "$DEFAULT_SALT_DIR/salt/elasticsearch/defaults.yaml" ]]; then
+        v2=$(egrep " +version: " "$DEFAULT_SALT_DIR/salt/elasticsearch/defaults.yaml" | awk -F: '{print $2}' | tr -d '[:space:]')
+      fi
+      local highest_es_version=$(compare_es_versions "$v1" "$v2")
+      local image=$i:$highest_es_version$IMAGE_TAG_SUFFIX
+      local sig_url=https://sigs.securityonion.net/es-$highest_es_version/$image.sig
+    else
+      # this is not an es container so use the so version for the version
+      local image=$i:$VERSION$IMAGE_TAG_SUFFIX
+      local sig_url=https://sigs.securityonion.net/$VERSION/$image.sig
+    fi
     # Pull down the trusted docker image
-    local image=$i:$VERSION$IMAGE_TAG_SUFFIX
     run_check_net_err \
     "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \
     "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 
     
     # Get signature
     run_check_net_err \
-    "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \
+    "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' $sig_url --output $SIGNPATH/$image.sig" \
     "Could not pull signature file for $image, please ensure connectivity to https://sigs.securityonion.net " \
     noretry >> "$LOG_FILE" 2>&1
     # Dump our hash values

salt/common/tools/sbin/so-log-check

@@ -125,6 +125,10 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tls handshake error"          # Docker registry container when new node comes onlines
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to get license information" # Logstash trying to contact ES before it's ready
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process already finished"     # Telegraf script finished just as the auto kill timeout kicked in
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No shard available"           # Typical error when making a query before ES has finished loading all indices
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|responded with status-code 503" # telegraf getting 503 from ES during startup
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process_cluster_event_timeout_exception" # logstash waiting for elasticsearch to start
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not configured for GeoIP"     # SO does not bundle the maxminddb with Zeek
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -150,6 +154,10 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error"              # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal"  # false positive (Open Canary logging out blank IP addresses)
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncing rule"                 # false positive (rule sync log line includes rule name which can contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request_unauthorized"         # false positive (login failures to Hydra result in an 'error' log) 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding index lifecycle policy" # false positive (elasticsearch policy names contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding ingest pipeline"       # false positive (elasticsearch ingest pipeline names contain 'error') 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|updating index template"      # false positive (elasticsearch index or template names contain 'error') 
 fi
 
 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -210,6 +218,8 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed"       # Detections: Exclude false positive due to automated testing
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors"                   # Detections: Not an actual error
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Initialized license manager"  # SOC log: before fields.status was changed to fields.licenseStatus
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|marked for removal"           # docker container getting recycled
 fi
 
 RESULT=0
@@ -248,6 +258,9 @@ exclude_log "agentstatus.log" # ignore this log since it tracks agents in error
 exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable
 exclude_log "/nsm/kafka/data/" # ignore Kafka data directory from log check.
 
+# Include Zeek reporter.log to detect errors after running known good pcap(s) through sensor
+echo "/nsm/zeek/spool/logger/reporter.log" >> /tmp/log_check_files
+
 for log_file in $(cat /tmp/log_check_files); do
     status "Checking log file $log_file"
     tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check

salt/common/tools/sbin_jinja/so-import-pcap

@@ -63,7 +63,7 @@ function status {
 function pcapinfo() {
   PCAP=$1
   ARGS=$2
-  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap $ARGS
+  docker run --rm -v "$PCAP:/input.pcap" --entrypoint capinfos {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} /input.pcap -ae $ARGS
 }
 
 function pcapfix() {

salt/docker/defaults.yaml

@@ -51,6 +51,14 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
+    'so-hydra':
+      final_octet: 30
+      port_bindings:
+        - 0.0.0.0:4444:4444
+        - 0.0.0.0:4445:4445
+      custom_bind_mounts: []
+      extra_hosts: []
+      extra_env: []
     'so-logstash':
       final_octet: 29
       port_bindings:
@@ -74,6 +82,7 @@ docker:
         - 443:443
         - 8443:8443
         - 7788:7788
+        - 7789:7789
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
@@ -191,6 +200,7 @@ docker:
       final_octet: 88
       port_bindings:
         - 0.0.0.0:9092:9092
+        - 0.0.0.0:29092:29092
         - 0.0.0.0:9093:9093
         - 0.0.0.0:8778:8778
       custom_bind_mounts: []

salt/docker/init.sls

@@ -20,41 +20,41 @@ dockergroup:
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.33-1
-      - docker-ce: 5:26.1.4-1~debian.12~bookworm
-      - docker-ce-cli: 5:26.1.4-1~debian.12~bookworm
-      - docker-ce-rootless-extras: 5:26.1.4-1~debian.12~bookworm
+      - containerd.io: 1.7.21-1
+      - docker-ce: 5:27.2.0-1~debian.12~bookworm
+      - docker-ce-cli: 5:27.2.0-1~debian.12~bookworm
+      - docker-ce-rootless-extras: 5:27.2.0-1~debian.12~bookworm
     - hold: True
     - update_holds: True
 {%    elif grains.oscodename == 'jammy' %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.33-1
-      - docker-ce: 5:26.1.4-1~ubuntu.22.04~jammy
-      - docker-ce-cli: 5:26.1.4-1~ubuntu.22.04~jammy
-      - docker-ce-rootless-extras: 5:26.1.4-1~ubuntu.22.04~jammy
+      - containerd.io: 1.7.21-1
+      - docker-ce: 5:27.2.0-1~ubuntu.22.04~jammy
+      - docker-ce-cli: 5:27.2.0-1~ubuntu.22.04~jammy
+      - docker-ce-rootless-extras: 5:27.2.0-1~ubuntu.22.04~jammy
     - hold: True
     - update_holds: True
 {%    else %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.33-1
-      - docker-ce: 5:26.1.4-1~ubuntu.20.04~focal
-      - docker-ce-cli: 5:26.1.4-1~ubuntu.20.04~focal
-      - docker-ce-rootless-extras: 5:26.1.4-1~ubuntu.20.04~focal
+      - containerd.io: 1.7.21-1
+      - docker-ce: 5:27.2.0-1~ubuntu.20.04~focal
+      - docker-ce-cli: 5:27.2.0-1~ubuntu.20.04~focal
+      - docker-ce-rootless-extras: 5:27.2.0-1~ubuntu.20.04~focal
     - hold: True
     - update_holds: True
-{% endif %}
+{%   endif %}
 {% else %}
 dockerheldpackages:
   pkg.installed:
     - pkgs:
-      - containerd.io: 1.6.33-3.1.el9
-      - docker-ce: 3:26.1.4-1.el9
-      - docker-ce-cli: 1:26.1.4-1.el9
-      - docker-ce-rootless-extras: 26.1.4-1.el9
+      - containerd.io: 1.7.21-3.1.el9
+      - docker-ce: 3:27.2.0-1.el9
+      - docker-ce-cli: 1:27.2.0-1.el9
+      - docker-ce-rootless-extras: 27.2.0-1.el9
     - hold: True
     - update_holds: True
 {% endif %}

salt/docker/soc_docker.yaml

@@ -45,6 +45,7 @@ docker:
     so-influxdb: *dockerOptions
     so-kibana: *dockerOptions
     so-kratos: *dockerOptions
+    so-hydra: *dockerOptions
     so-logstash: *dockerOptions
     so-nginx: *dockerOptions
     so-nginx-fleet-node: *dockerOptions

salt/elastalert/soc_elastalert.yaml

@@ -1,6 +1,6 @@
 elastalert:
   enabled:
-    description: You can enable or disable Elastalert.
+    description: Enables or disables the ElastAlert 2 process. This process is critical for ensuring alerts arrive in SOC, and for outbound notification delivery.
     helpLink: elastalert.html
   alerter_parameters:
     title: Custom Configuration Parameters

salt/elastic-fleet-package-registry/soc_elastic-fleet-package-registry.yaml

@@ -1,4 +1,4 @@
 elastic_fleet_package_registry:
   enabled:
-    description: You can enable or disable Elastic Fleet Package Registry.
+    description: Enables or disables the Fleet package registry process. This process must remain enabled to allow Elastic Agent packages to be updated.
     advanced: True

salt/elasticagent/enabled.sls

@@ -8,7 +8,6 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
 
-
 include:
   - elasticagent.config
   - elasticagent.sostatus

salt/elasticagent/soc_elasticagent.yaml

@@ -0,0 +1,4 @@
+elasticagent:
+  enabled:
+    description: Enables or disables the Elastic Agent process. This process must remain enabled to allow collection of node events.
+    advanced: True

salt/elasticfleet/config.sls

@@ -30,6 +30,7 @@ elasticfleet_sbin:
     - user: 947
     - group: 939
     - file_mode: 755
+    - show_changes: False
 
 elasticfleet_sbin_jinja:
   file.recurse:
@@ -41,6 +42,7 @@ elasticfleet_sbin_jinja:
     - template: jinja
     - exclude_pat:
       - so-elastic-fleet-package-upgrade # exclude this because we need to watch it for changes
+    - show_changes: False
 
 eaconfdir:
   file.directory:
@@ -63,6 +65,14 @@ eastatedir:
     - group: 939
     - makedirs: True
 
+custommappingsdir:
+  file.directory:
+    - name: /nsm/custom-mappings
+    - user: 947
+    - group: 939
+    - makedirs: True
+
+
 eapackageupgrade:
   file.managed:
     - name: /usr/sbin/so-elastic-fleet-package-upgrade
@@ -73,6 +83,56 @@ eapackageupgrade:
     - template: jinja
 
 {%   if GLOBALS.role != "so-fleet" %}
+   
+{% if not GLOBALS.airgap %}
+soresourcesrepoclone:
+  git.latest:
+    - name: https://github.com/Security-Onion-Solutions/securityonion-resources.git
+    - target: /nsm/securityonion-resources
+    - rev: 'main'
+    - depth: 1
+    - force_reset: True
+{% endif %}
+
+elasticdefendconfdir:
+  file.directory:
+    - name: /opt/so/conf/elastic-fleet/defend-exclusions/rulesets
+    - user: 947
+    - group: 939
+    - makedirs: True
+  
+elasticdefenddisabled:
+  file.managed:
+    - name: /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml
+    - source: salt://elasticfleet/files/soc/elastic-defend-disabled-filters.yaml
+    - user: 947
+    - group: 939
+    - mode: 600
+
+elasticdefendcustom:
+  file.managed:
+    - name: /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters-raw
+    - source: salt://elasticfleet/files/soc/elastic-defend-custom-filters.yaml
+    - user: 947
+    - group: 939
+    - mode: 600
+
+{% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
+{%   set ap = "present" %}
+{% else %}
+{%   set ap = "absent" %}
+{% endif %}
+cron-elastic-defend-filters:
+  cron.{{ap}}:
+    - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
+    - identifier: elastic-defend-filters
+    - user: root
+    - minute: '0'
+    - hour: '3'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
 eaintegrationsdir:
   file.directory:
     - name: /opt/so/conf/elastic-fleet/integrations
@@ -87,6 +147,7 @@ eadynamicintegration:
     - user: 947
     - group: 939
     - template: jinja
+    - show_changes: False
 
 eaintegration:
   file.recurse:
@@ -94,6 +155,7 @@ eaintegration:
     - source: salt://elasticfleet/files/integrations
     - user: 947
     - group: 939
+    - show_changes: False
 
 eaoptionalintegrationsdir:
   file.directory:

salt/elasticfleet/defaults.yaml

@@ -8,6 +8,10 @@ elasticfleet:
       endpoints_enrollment: ''
       es_token: ''
       grid_enrollment: ''
+    defend_filters:
+      enable_auto_configuration: False
+    subscription_integrations: False
+    auto_upgrade_integrations: False
   logging:
     zeek:
       excluded:
@@ -30,89 +34,20 @@ elasticfleet:
         - stderr
         - stdout
   packages:
-    - apache
-    - auditd
-    - auth0
-    - aws
-    - azure
-    - barracuda
-    - carbonblack_edr
-    - cef
-    - checkpoint
-    - cisco_asa
-    - cisco_duo
-    - cisco_ftd
-    - cisco_ios
-    - cisco_ise
-    - cisco_meraki
-    - cisco_umbrella
-    - citrix_adc
-    - citrix_waf
-    - cloudflare
-    - crowdstrike
-    - darktrace
     - elastic_agent
     - elasticsearch
     - endpoint
-    - f5_bigip
-    - fim
-    - fireeye
     - fleet_server
-    - fortinet
-    - fortinet_fortigate
-    - gcp
-    - github
-    - google_workspace
     - http_endpoint
     - httpjson
-    - iis
-    - journald
-    - juniper
-    - juniper_srx
-    - kafka_log
-    - lastpass
     - log
-    - m365_defender
-    - microsoft_defender_endpoint
-    - microsoft_dhcp
-    - microsoft_sqlserver
-    - mimecast
-    - mysql
-    - netflow
-    - nginx
-    - o365
-    - okta
     - osquery_manager
-    - panw
-    - pfsense
-    - proofpoint_tap
-    - pulse_connect_secure
     - redis
-    - sentinel_one
-    - snort
-    - snyk
-    - sonicwall_firewall
-    - sophos
-    - sophos_central
-    - symantec_endpoint
     - system
     - tcp
-    - tenable_io
-    - tenable_sc
-    - ti_abusech
-    - ti_anomali
-    - ti_cybersixgill
-    - ti_misp
-    - ti_otx
-    - ti_recordedfuture
-    - ti_threatq
     - udp
-    - vsphere
     - windows
     - winlog
-    - zscaler_zia
-    - zscaler_zpa
-    - 1password
   optional_integrations:
     sublime_platform:
       enabled_nodes: []

salt/elasticfleet/enabled.sls

@@ -17,10 +17,12 @@ include:
   - elasticfleet.sostatus
   - ssl
 
+{% if grains.role not in ['so-fleet'] %}
 # Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready
 wait_for_elasticsearch_elasticfleet:
   cmd.run:
     - name: so-elasticsearch-wait
+{% endif %}
 
 # If enabled, automatically update Fleet Logstash Outputs
 {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %}
@@ -141,7 +143,26 @@ so-elastic-fleet-integrations:
 so-elastic-agent-grid-upgrade:
   cmd.run:
     - name: /usr/sbin/so-elastic-agent-grid-upgrade
-    - retry: True
+    - retry:
+        attempts: 12
+        interval: 5
+
+so-elastic-fleet-integration-upgrade:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
+
+so-elastic-fleet-addon-integrations:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-optional-integrations-load
+
+{%   if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
+so-elastic-defend-manage-filters-file-watch:
+  cmd.run:
+    - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
+    - onchanges:
+      - file: elasticdefendcustom
+      - file: elasticdefenddisabled
+{%    endif %}
 {%  endif %}
 
 delete_so-elastic-fleet_so-status.disabled:

salt/elasticfleet/files/integrations-dynamic/grid-nodes_general/kratos-logs.json

@@ -0,0 +1,46 @@
+{%- set identities = salt['sqlite3.fetch']('/nsm/kratos/db/db.sqlite', 'SELECT id, json_extract(traits, "$.email") as email FROM identities;') -%}
+{%- set valid_identities = false -%}
+{%- if identities -%}
+  {%- set valid_identities = true -%}
+  {%- for id, email in identities -%}
+    {%- if not id or not email -%}
+      {%- set valid_identities = false -%}
+      {%- break -%}
+    {%- endif -%}
+  {%- endfor -%}
+{%- endif -%}
+{
+  "package": {
+    "name": "log",
+    "version": ""
+  },
+  "name": "kratos-logs",
+  "namespace": "so",
+  "description": "Kratos logs",
+  "policy_id": "so-grid-nodes_general",
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.logs": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/opt/so/log/kratos/kratos.log"
+            ],
+            "data_stream.dataset": "kratos",
+            "tags": ["so-kratos"],
+            {%- if valid_identities -%}
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos\n- if:\n    has_fields:\n      - identity_id\n  then:{% for id, email in identities %}\n    - if:\n        equals:\n          identity_id: \"{{ id }}\"\n      then:\n        - add_fields:\n            target: ''\n            fields:\n              user.name: \"{{ email }}\"{% endfor %}",
+            {%- else -%}
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos",
+            {%- endif -%}
+            "custom": "pipeline: kratos"
+          }
+        }
+      }
+    }
+  },
+  "force": true
+}
+

salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json

@@ -3,25 +3,30 @@
 	"namespace": "default",
 	"description": "",
 	"package": {
-		"name": "endpoint",
-		"title": "Elastic Defend",
-		"version": "8.14.0"
+	  "name": "endpoint",
+	  "title": "Elastic Defend",
+	  "version": "8.17.0",
+	  "requires_root": true
 	},
 	"enabled": true,
 	"policy_id": "endpoints-initial",
-	"inputs": [{
-		"type": "ENDPOINT_INTEGRATION_CONFIG",
+	"vars": {},
+	"inputs": [
+	  {
+		"type": "endpoint",
 		"enabled": true,
-		"streams": [],
 		"config": {
-			"_config": {
-				"value": {
-					"type": "endpoint",
-					"endpointConfig": {
-						"preset": "DataCollection"
-					}
-				}
+		  "integration_config": {
+			"value": {
+			  "type": "endpoint",
+			  "endpointConfig": {
+				"preset": "DataCollection"
+			  }
 			}
-		}
-	}]
-}
+		  }
+		},
+		"streams": []
+	  }
+	]
+  }
+  

salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json

@@ -3,9 +3,9 @@
     "name": "log",
     "version": ""
   },
-  "name": "kratos-logs",
+  "name": "hydra-logs",
   "namespace": "so",
-  "description": "Kratos logs",
+  "description": "Hydra logs",
   "policy_id": "so-grid-nodes_general",
   "inputs": {
     "logs-logfile": {
@@ -15,12 +15,12 @@
           "enabled": true,
           "vars": {
             "paths": [
-              "/opt/so/log/kratos/kratos.log"
+              "/opt/so/log/hydra/hydra.log"
             ],
-            "data_stream.dataset": "kratos",
-            "tags": ["so-kratos"],
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos",
-            "custom": "pipeline: kratos"
+            "data_stream.dataset": "hydra",
+            "tags": ["so-hydra"],
+            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: hydra",
+            "custom": "pipeline: hydra"
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json

@@ -20,7 +20,7 @@
             ],
             "data_stream.dataset": "import",
             "custom": "",
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-1.59.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-1.45.1\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-1.59.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-1.59.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-1.45.1\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-1.67.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-2.5.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-1.67.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-1.67.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-2.5.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
             "tags": [
               "import"
             ]

salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json

@@ -0,0 +1,35 @@
+{
+  "package": {
+    "name": "log",
+    "version": ""
+  },
+  "name": "so-ip-mappings",
+  "namespace": "so",
+  "description": "IP Description mappings",
+  "policy_id": "so-grid-nodes_general",
+  "vars": {},
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.logs": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/nsm/custom-mappings/ip-descriptions.csv"
+            ],
+            "data_stream.dataset": "hostnamemappings",
+            "tags": [
+              "so-ip-mappings"
+            ],
+            "processors": "- decode_csv_fields:\n    fields:\n      message: decoded.csv\n    separator: \",\"\n    ignore_missing: false\n    overwrite_keys: true\n    trim_leading_space: true\n    fail_on_error: true\n\n- extract_array:\n    field: decoded.csv\n    mappings:\n      so.ip_address: '0'\n      so.description: '1'\n\n- script:\n    lang: javascript\n    source: >\n      function process(event) {\n          var ip = event.Get('so.ip_address');\n          var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n          if (!validIpRegex.test(ip)) {\n              event.Cancel();\n          }\n      }\n- fingerprint:\n    fields: [\"so.ip_address\"]\n    target_field: \"@metadata._id\"\n",
+            "custom": ""
+          }
+        }
+      }
+    }
+  },
+  "force": true
+}
+
+

salt/elasticfleet/files/integrations/grid-nodes_general/syslog-udp-514.json

@@ -11,7 +11,7 @@
     "udp-udp": {
       "enabled": true,
       "streams": {
-        "udp.generic": {
+        "udp.udp": {
           "enabled": true,
           "vars": {
             "listen_address": "0.0.0.0",
@@ -20,11 +20,13 @@
             "pipeline": "syslog",
             "max_message_size": "10KiB",
             "keep_null": false,
-            "processors": "- add_fields:\n    target: event\n    fields: \n      module: syslog\n",
+            "processors": "- add_fields:\n    target: event\n    fields: \n      module: syslog",
             "tags": [
               "syslog"
             ],
-            "syslog_options": "field: message\n#format: auto\n#timezone: Local"
+            "syslog_options": "field: message\n#format: auto\n#timezone: Local\n",
+            "preserve_original_event": false,
+            "custom": ""
           }
         }
       }

salt/elasticfleet/files/integrations/grid-nodes_general/system-grid-nodes.json

@@ -31,7 +31,8 @@
             ],
             "tags": [
               "so-grid-node"
-            ]
+            ],
+            "processors": "- if:\n    contains:\n      message: \"salt-minion\"\n  then: \n    - dissect:\n        tokenizer: \"%{} %{} %{} %{} %{} %{}: [%{log.level}] %{*}\"\n        field: \"message\"\n        trim_values: \"all\"\n        target_prefix: \"\"\n    - drop_event:\n        when:\n          equals:\n            log.level: \"INFO\""
           }
         }
       }

salt/elasticfleet/files/soc/elastic-defend-custom-filters.yaml

@@ -0,0 +1,27 @@
+title: 'Template 1'
+id: 'This needs to be a UUIDv4 id - https://www.uuidgenerator.net/version4'
+description: 'Short description detailing what this rule is filtering and why.'
+references: 'Relevant urls, etc'
+author: '@SecurityOnion'
+date: 'MM/DD/YY'
+event_type: 'dns_query'
+filter_type: 'exclude'
+filter:
+  selection_1:
+    TargetField: 'QueryName'
+    Condition: 'end with'
+    Pattern: '.thawte.com'
+---
+title: 'Template 2'
+id: 'This needs to be a UUIDv4 id - https://www.uuidgenerator.net/version4'
+description: 'Short description detailing what this rule is filtering and why.'
+references: 'Relevant urls, etc'
+author: '@SecurityOnion'
+date: 'MM/DD/YY'
+event_type: 'process_creation'
+filter_type: 'exclude'
+filter:
+  selection_1:
+    TargetField: 'ParentImage'
+    Condition: 'is'
+    Pattern: 'C:\Windows\Microsoft.NET\Framework\v4.0.30319\ngentask.exe'

salt/elasticfleet/files/soc/elastic-defend-disabled-filters.yaml

@@ -0,0 +1,3 @@
+'9EDAA51C-BB12-49D9-8748-2B61371F2E7D':
+  Date: '10/10/2024'
+  Notes: 'Example Disabled Filter - Leave this entry here, just copy and paste as needed.'

salt/elasticfleet/integration-defaults.map.jinja

@@ -0,0 +1,133 @@
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+  or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+  this file except in compliance with the Elastic License 2.0. #}
+
+
+{% import_json '/opt/so/state/esfleet_package_components.json' as ADDON_PACKAGE_COMPONENTS %}
+{% import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+
+{% set CORE_ESFLEET_PACKAGES = ELASTICFLEETDEFAULTS.get('elasticfleet', {}).get('packages', {}) %}
+{% set ADDON_INTEGRATION_DEFAULTS = {} %}
+
+{# Some fleet integrations don't follow the standard naming convention #}
+{% set WEIRD_INTEGRATIONS = {
+    'awsfirehose.logs': 'awsfirehose',
+    'awsfirehose.metrics': 'aws.cloudwatch',
+    'cribl.logs': 'cribl',
+    'sentinel_one_cloud_funnel.logins': 'sentinel_one_cloud_funnel.login',
+    'azure_application_insights.app_insights': 'azure.app_insights',
+    'azure_application_insights.app_state': 'azure.app_state',
+    'azure_billing.billing': 'azure.billing',
+    'azure_functions.metrics': 'azure.function',
+    'azure_metrics.compute_vm_scaleset': 'azure.compute_vm_scaleset',
+    'azure_metrics.compute_vm': 'azure.compute_vm',
+    'azure_metrics.container_instance': 'azure.container_instance',
+    'azure_metrics.container_registry': 'azure.container_registry',
+    'azure_metrics.container_service': 'azure.container_service',
+    'azure_metrics.database_account': 'azure.database_account',
+    'azure_metrics.monitor': 'azure.monitor',
+    'azure_metrics.storage_account': 'azure.storage_account',
+    'azure_openai.metrics': 'azure.open_ai',
+    'beat.state': 'beats.stack_monitoring.state',
+    'beat.stats': 'beats.stack_monitoring.stats',
+    'enterprisesearch.health': 'enterprisesearch.stack_monitoring.health',
+    'enterprisesearch.stats': 'enterprisesearch.stack_monitoring.stats',
+    'kibana.cluster_actions': 'kibana.stack_monitoring.cluster_actions',
+    'kibana.cluster_rules': 'kibana.stack_monitoring.cluster_rules',
+    'kibana.node_actions': 'kibana.stack_monitoring.node_actions',
+    'kibana.node_rules': 'kibana.stack_monitoring.node_rules',
+    'kibana.stats': 'kibana.stack_monitoring.stats',
+    'kibana.status': 'kibana.stack_monitoring.status',
+    'logstash.node_cel': 'logstash.stack_monitoring.node',
+    'logstash.node_stats': 'logstash.stack_monitoring.node_stats',
+    'synthetics.browser': 'synthetics-browser',
+    'synthetics.browser_network': 'synthetics-browser.network',
+    'synthetics.browser_screenshot': 'synthetics-browser.screenshot',
+    'synthetics.http': 'synthetics-http',
+    'synthetics.icmp': 'synthetics-icmp',
+    'synthetics.tcp': 'synthetics-tcp'
+   } %}
+
+{% for pkg in ADDON_PACKAGE_COMPONENTS %}
+{%   if pkg.name in CORE_ESFLEET_PACKAGES %}
+{#     skip core integrations #}
+{%   elif pkg.name not in CORE_ESFLEET_PACKAGES %}
+{#     generate defaults for each integration #}
+{%     if pkg.es_index_patterns is defined and pkg.es_index_patterns is not none %}
+{%       for pattern in pkg.es_index_patterns %}
+{%         if "metrics-" in pattern.name %}
+{%           set integration_type = "metrics-" %}
+{%         elif "logs-" in pattern.name %}
+{%           set integration_type = "logs-" %}
+{%         else %}
+{%           set integration_type = "" %}
+{%         endif %}
+{%           set component_name = pkg.name ~ "." ~ pattern.title %}
+{#           fix weirdly named components #}
+{%           if component_name in WEIRD_INTEGRATIONS %}
+{%             set component_name = WEIRD_INTEGRATIONS[component_name] %}
+{%           endif %}
+{#           component_name_x maintains the functionality of merging local pillar changes with generated 'defaults' via SOC UI #}
+{%           set component_name_x = component_name.replace(".","_x_") %}
+{#           pillar overrides/merge expects the key names to follow the naming in elasticsearch/defaults.yaml eg. so-logs-1password_x_item_usages . The _x_ is replaced later on in elasticsearch/template.map.jinja #}
+{%           set integration_key = "so-" ~ integration_type ~ component_name_x %}
+
+{#           Default integration settings #}
+{%           set integration_defaults = {
+              "index_sorting": false,
+              "index_template": {
+                  "composed_of": [integration_type ~ component_name ~ "@package", integration_type ~ component_name ~ "@custom", "so-fleet_integrations.ip_mappings-1", "so-fleet_globals-1", "so-fleet_agent_id_verification-1"],
+                  "data_stream": {
+                      "allow_custom_routing": false,
+                      "hidden": false
+                  },
+                  "ignore_missing_component_templates": [integration_type ~ component_name ~ "@custom"],
+                  "index_patterns": [pattern.name],
+                  "priority": 501,
+                  "template": {
+                      "settings": {
+                          "index": {
+                              "lifecycle": {"name": "so-" ~ integration_type ~ component_name ~ "-logs"},
+                              "number_of_replicas": 0
+                          }
+                      }
+                  }
+              },
+              "policy": {
+                  "phases": {
+                      "cold": {
+                          "actions": {
+                              "set_priority": {"priority": 0}
+                          },
+                          "min_age": "60d"
+                      },
+                      "delete": {
+                          "actions": {
+                              "delete": {}
+                          },
+                          "min_age": "365d"
+                      },
+                      "hot": {
+                          "actions": {
+                              "rollover": {
+                                  "max_age": "30d",
+                                  "max_primary_shard_size": "50gb"
+                              },
+                              "set_priority": {"priority": 100}
+                          },
+                          "min_age": "0ms"
+                      },
+                      "warm": {
+                          "actions": {
+                              "set_priority": {"priority": 50}
+                          },
+                          "min_age": "30d"
+                      }
+                  }
+              }
+          } %}
+{%         do ADDON_INTEGRATION_DEFAULTS.update({integration_key: integration_defaults}) %}
+{%       endfor %}
+{%     endif %}
+{%   endif %}
+{% endfor %}

salt/elasticfleet/soc_elasticfleet.yaml

@@ -1,6 +1,6 @@
 elasticfleet:
   enabled:
-    description: You can enable or disable Elastic Fleet.
+    description: Enables or disables the Elastic Fleet process. This process is critical for managing Elastic Agents.
     advanced: True
     helpLink: elastic-fleet.html
   enable_manager_output:
@@ -9,6 +9,24 @@ elasticfleet:
     global: True
     forcedType: bool
     helpLink: elastic-fleet.html
+  files:
+    soc:
+      elastic-defend-disabled-filters__yaml:
+        title: Disabled Elastic Defend filters
+        description: Enter the ID of the filter that should be disabled.
+        syntax: yaml
+        file: True
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+      elastic-defend-custom-filters__yaml:
+        title: Custom Elastic Defend filters
+        description: Enter custom filters seperated by ---
+        syntax: yaml
+        file: True
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
   logging:
     zeek:
       excluded:
@@ -16,6 +34,22 @@ elasticfleet:
         forcedType: "[]string"
         helpLink: zeek.html
   config:
+    defend_filters:
+      enable_auto_configuration:
+        description: Enable auto-configuration and management of the Elastic Defend Exclusion filters.
+        global: True
+        helpLink: elastic-fleet.html
+        advanced: True
+    subscription_integrations:
+      description: Enable the installation of integrations that require an Elastic license.
+      global: True
+      forcedType: bool
+      helpLink: elastic-fleet.html
+    auto_upgrade_integrations:
+      description: Enables or disables automatically upgrading Elastic Agent integrations.
+      global: True
+      forcedType: bool
+      helpLink: elastic-fleet.html
     server:
       custom_fqdn:
         description: Custom FQDN for Agents to connect to. One per line.

salt/elasticfleet/tools/sbin/so-elastic-defend-manage-filters.py

@@ -0,0 +1,251 @@
+from datetime import datetime
+import sys
+import getopt
+from so_elastic_defend_filters_helper import *
+import logging
+
+logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+# Define mappings for Target Field, Event Type, Conditions
+TARGET_FIELD_MAPPINGS = {
+    "Image": "process.executable",
+    "ParentImage": "process.parent.executable",
+    "CommandLine": "process.command_line",
+    "ParentCommandLine": "process.parent.command_line",
+    "DestinationHostname": "destination.domain",
+    "QueryName": "dns.question.name",
+    "DestinationIp": "destination.ip",
+    "TargetObject": "registry.path",
+    "TargetFilename": "file.path"
+}
+
+DATASET_MAPPINGS = {
+    "process_create": "endpoint.events.process",
+    "network_connection": "endpoint.events.network",
+    "file_create": "endpoint.events.file",
+    "file_delete": "endpoint.events.file",
+    "registry_event": "endpoint.events.registry",
+    "dns_query": "endpoint.events.network"
+}
+
+CONDITION_MAPPINGS = {
+    "is": ("included", "match"),
+    "end with": ("included", "wildcard"),
+    "begin with": ("included", "wildcard"),
+    "contains": ("included", "wildcard")
+}
+
+# Extract entries for a rule
+def extract_entries(data, event_type):
+    entries = []
+    filter_data = data.get('filter', {})
+    for value in filter_data.values():
+        target_field = TARGET_FIELD_MAPPINGS.get(value.get('TargetField', ''))
+        condition = value.get('Condition', '')
+        pattern = value.get('Pattern', '')
+
+        if condition not in CONDITION_MAPPINGS:
+            logging.error(f"Invalid condition: {condition}")
+
+        # Modify the pattern based on the condition
+        pattern = modify_pattern(condition, pattern)
+
+        operator, match_type = CONDITION_MAPPINGS[condition]
+
+        entries.append({
+            "field": target_field,
+            "operator": operator,
+            "type": match_type,
+            "value": pattern
+        })
+
+    # Add the event.dataset entry from DATASET_MAPPINGS
+    dataset_value = DATASET_MAPPINGS.get(event_type, '')
+    if dataset_value:
+        entries.append({
+            "field": "event.dataset",
+            "operator": "included",
+            "type": "match",
+            "value": dataset_value
+        })
+    else:
+        logging.error(f"No dataset mapping found for event_type: {event_type}")
+
+    return entries
+
+# Build the JSON
+def build_json_entry(entries, guid, event_type, context):
+    return {
+        "comments": [],
+        "entries": entries,
+        "item_id": guid,
+        "name": f"SO - {event_type} - {guid}",
+        "description": f"{context}\n\n  <<- Note: This filter is managed by Security Onion. ->>",
+        "namespace_type": "agnostic",
+        "tags": ["policy:all"],
+        "type": "simple",
+        "os_types": ["windows"],
+        "entries": entries
+    }
+
+# Check to see if the rule is disabled
+# If it is, make sure it is not active
+def disable_check(guid, disabled_rules, username, password):
+    if guid in disabled_rules:
+        logging.info(f"Rule {guid} is in the disabled rules list, confirming that is is actually disabled...")
+        existing_rule = api_request("GET", guid, username, password)
+
+        if existing_rule:
+            if api_request("DELETE", guid, username, password):
+                logging.info(f"Successfully deleted rule {guid}")
+                return True, "deleted"
+            else:
+                logging.error(f"Error deleting rule {guid}.")
+                return True, "Error deleting"
+        return True, "NOP"
+    return False, None
+
+def modify_pattern(condition, pattern):
+    """
+    Modify the pattern based on the condition.
+    - 'end with': Add '*' to the beginning of the pattern.
+    - 'begin with': Add '*' to the end of the pattern.
+    - 'contains': Add '*' to both the beginning and end of the pattern.
+    """
+    if isinstance(pattern, list):
+        # Apply modification to each pattern in the list if it's a list of patterns
+        return [modify_pattern(condition, p) for p in pattern]
+    
+    if condition == "end with":
+        return f"*{pattern}"
+    elif condition == "begin with":
+        return f"{pattern}*"
+    elif condition == "contains":
+        return f"*{pattern}*"
+    return pattern
+
+
+def process_rule_update_or_create(guid, json_entry, username, password):
+    existing_rule = api_request("GET", guid, username, password)
+
+    if existing_rule:
+        existing_rule_data = extract_relevant_fields(existing_rule)
+        new_rule_data = extract_relevant_fields(json_entry)
+        if generate_hash(existing_rule_data) != generate_hash(new_rule_data):
+            logging.info(f"Updating rule {guid}")
+            json_entry.pop("list_id", None)
+            api_request("PUT", guid, username, password, json_data=json_entry)
+            return "updated"
+        logging.info(f"Rule {guid} is up to date.")
+        return "no_change"
+    else:
+        logging.info(f"Creating new rule {guid}")
+        json_entry["list_id"] = "endpoint_event_filters"
+        api_request("POST", guid, username, password, json_data=json_entry)
+        return "new"
+
+# Main function for processing rules
+def process_rules(yaml_files, disabled_rules, username, password):
+    stats = {"rule_count": 0, "new": 0, "updated": 0, "no_change": 0, "disabled": 0, "deleted": 0}
+    for data in yaml_files:
+        logging.info(f"Processing rule: {data.get('id', '')}")
+        event_type = data.get('event_type', '')
+        guid = data.get('id', '')
+        dataset = DATASET_MAPPINGS.get(event_type, '')
+        context = data.get('description', '')
+
+        rule_deleted, state = disable_check(guid, disabled_rules, username, password)
+        if rule_deleted:
+            stats["disabled"] += 1
+            if state == "deleted":
+                stats["deleted"] += 1
+            continue
+
+        # Extract entries and build JSON
+        entries = extract_entries(data, event_type)
+        json_entry = build_json_entry(entries, guid, event_type, context)
+
+        # Process rule creation or update
+        status = process_rule_update_or_create(guid, json_entry, username, password)
+
+        stats[status] += 1
+        stats["rule_count"] += 1
+    return stats
+
+def parse_args(argv):
+    try:
+        opts, args = getopt.getopt(argv, "i:d:c:f:", ["input=", "disabled=", "credentials=", "flags_file="])
+    except getopt.GetoptError:
+        print("Usage: python so-elastic-defend-manage-filters.py -c <credentials_file> -d <disabled_file> -i <folder_of_yaml_files> [-f <flags_file>]")
+        sys.exit(2)
+    return opts
+
+def load_flags(file_path):
+    with open(file_path, 'r') as flags_file:
+        return flags_file.read().splitlines()
+
+def validate_inputs(credentials_file, disabled_file, yaml_directories):
+    if not credentials_file or not disabled_file or not yaml_directories:
+        print("Usage: python so-elastic-defend-manage-filters.py -c <credentials_file> -d <disabled_file> -i <folder_of_yaml_files> [-f <flags_file>]")
+        sys.exit(2)
+
+def main(argv):
+    credentials_file = ""
+    disabled_file = ""
+    yaml_directories = []
+
+    opts = parse_args(argv)
+
+    for opt, arg in opts:
+        if opt in ("-c", "--credentials"):
+            credentials_file = arg
+        elif opt in ("-d", "--disabled"):
+            disabled_file = arg
+        elif opt in ("-i", "--input"):
+            yaml_directories.append(arg)
+        elif opt in ("-f", "--flags_file"):
+            flags = load_flags(arg)
+            return main(argv + flags)
+
+    timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    logging.info(f"\n{timestamp}")
+
+    validate_inputs(credentials_file, disabled_file, yaml_directories)
+
+    credentials = load_credentials(credentials_file)
+    if not credentials:
+        raise Exception("Failed to load credentials")
+
+    username, password = extract_auth_details(credentials)
+    if not username or not password:
+        raise Exception("Invalid credentials format")
+
+    custom_rules_input = '/opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters-raw'
+    custom_rules_output = '/opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters'
+    prepare_custom_rules(custom_rules_input, custom_rules_output)
+    disabled_rules = load_disabled(disabled_file)
+
+    total_stats = {"rule_count": 0, "new": 0, "updated": 0, "no_change": 0, "disabled": 0, "deleted": 0}
+
+    for yaml_dir in yaml_directories:
+        yaml_files = load_yaml_files(yaml_dir)
+        stats = process_rules(yaml_files, disabled_rules, username, password)
+
+        for key in total_stats:
+            total_stats[key] += stats[key]
+
+    logging.info(f"\nProcessing Summary")
+    logging.info(f" - Total processed rules: {total_stats['rule_count']}")
+    logging.info(f" - New rules: {total_stats['new']}")
+    logging.info(f" - Updated rules: {total_stats['updated']}")
+    logging.info(f" - Disabled rules: {total_stats['deleted']}")
+    logging.info(f" - Rules with no changes: {total_stats['no_change']}")
+    logging.info(f"Rule status Summary")
+    logging.info(f" - Active rules: {total_stats['rule_count'] - total_stats['disabled']}")
+    logging.info(f" - Disabled rules: {total_stats['disabled']}")
+    timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    logging.info(f"Execution completed at: {timestamp}")
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

salt/elasticfleet/tools/sbin/so-elastic-fleet-common

@@ -97,11 +97,84 @@ elastic_fleet_package_install() {
     curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  -d '{"force":true}' "localhost:5601/api/fleet/epm/packages/$PKG/$VERSION"
 }
 
+elastic_fleet_bulk_package_install() {
+    BULK_PKG_LIST=$1
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d@$1 "localhost:5601/api/fleet/epm/packages/_bulk"
+}
+
 elastic_fleet_package_is_installed() {
     PACKAGE=$1
     curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.status'
 }
 
+elastic_fleet_installed_packages() {
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' -H 'Content-Type: application/json' "localhost:5601/api/fleet/epm/packages/installed?perPage=500"
+}
+
+elastic_fleet_agent_policy_ids() {
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].id
+    if [ $? -ne 0 ]; then
+        echo "Error: Failed to retrieve agent policies."
+        exit 1
+    fi
+}
+
+elastic_fleet_agent_policy_names() {
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].name
+    if [ $? -ne 0 ]; then
+        echo "Error: Failed to retrieve agent policies."
+        exit 1
+    fi
+}
+
+elastic_fleet_integration_policy_names() {
+    AGENT_POLICY=$1
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r .item.package_policies[].name
+    if [ $? -ne 0 ]; then
+        echo "Error: Failed to retrieve integrations for '$AGENT_POLICY'."
+        exit 1
+    fi
+}
+
+elastic_fleet_integration_policy_package_name() {
+    AGENT_POLICY=$1
+    INTEGRATION=$2
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.name'
+    if [ $? -ne 0 ]; then
+        echo "Error: Failed to retrieve package name for '$INTEGRATION' in '$AGENT_POLICY'."
+        exit 1
+    fi
+}
+
+elastic_fleet_integration_policy_package_version() {
+    AGENT_POLICY=$1
+    INTEGRATION=$2
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version'
+    if [ $? -ne 0 ]; then
+        echo "Error: Failed to retrieve package version for '$INTEGRATION' in '$AGENT_POLICY'."
+        exit 1
+    fi
+}
+
+elastic_fleet_integration_id() {
+    AGENT_POLICY=$1
+    INTEGRATION=$2
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .id'
+    if [ $? -ne 0 ]; then
+        echo "Error: Failed to retrieve integration ID for '$INTEGRATION' in '$AGENT_POLICY'."
+        exit 1
+    fi
+}
+
+elastic_fleet_integration_policy_dryrun_upgrade() {
+    INTEGRATION_ID=$1
+    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -H "Content-Type: application/json" -H 'kbn-xsrf: true' -L -X POST "localhost:5601/api/fleet/package_policies/upgrade/dryrun" -d "{\"packagePolicyIds\":[\"$INTEGRATION_ID\"]}"
+    if [ $? -ne 0 ]; then
+        echo "Error: Failed to complete dry run for '$INTEGRATION_ID'."
+        exit 1
+    fi
+}
+
 elastic_fleet_policy_create() {
 
     NAME=$1

salt/elasticfleet/tools/sbin/so-elastic-fleet-package-list

@@ -10,6 +10,6 @@
 SESSIONCOOKIE=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
 
 # List configured package policies
-curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages" -H 'kbn-xsrf: true' | jq
+curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/epm/packages?prerelease=true" -H 'kbn-xsrf: true' | jq
 
 echo

salt/elasticfleet/tools/sbin/so_elastic_defend_filters_helper.py

@@ -0,0 +1,128 @@
+
+import hashlib
+import os
+import json
+import yaml
+import requests
+from requests.auth import HTTPBasicAuth
+import shutil
+
+# Extract 'entries', 'description' and 'os_types' fields
+def extract_relevant_fields(filter):
+    return {
+        'entries': filter.get('entries', []),
+        'description': filter.get('description', '')
+    }
+
+# Sort for consistency, so that a hash can be generated
+def sorted_data(value):
+    if isinstance(value, dict):
+        # Recursively sort the dictionary by key
+        return {k: sorted_data(v) for k, v in sorted(value.items())}
+    elif isinstance(value, list):
+        # Sort lists; for dictionaries, sort by a specific key
+        return sorted(value, key=lambda x: tuple(sorted(x.items())) if isinstance(x, dict) else x)
+    return value
+
+# Generate a hash based on sorted relevant fields
+def generate_hash(data):
+    sorted_data_string = json.dumps(sorted_data(data), sort_keys=True)
+    return hashlib.sha256(sorted_data_string.encode('utf-8')).hexdigest()
+
+# Load Elasticsearch credentials from the config file
+def load_credentials(config_path):
+    with open(config_path, 'r') as file:
+        for line in file:
+            if line.startswith("user"):
+                credentials = line.split('=', 1)[1].strip().strip('"')
+                return credentials
+    return None
+
+# Extract username and password from credentials
+def extract_auth_details(credentials):
+    if ':' in credentials:
+        return credentials.split(':', 1)
+    return None, None
+
+# Generalized API request function
+def api_request(method, guid, username, password, json_data=None):
+    headers = {
+        'kbn-xsrf': 'true',
+        'Content-Type': 'application/json'
+    }
+    auth = HTTPBasicAuth(username, password)
+
+    if method == "POST":
+        url = "http://localhost:5601/api/exception_lists/items?namespace_type=agnostic"
+    else:
+        url = f"http://localhost:5601/api/exception_lists/items?item_id={guid}&namespace_type=agnostic"
+
+    response = requests.request(method, url, headers=headers, auth=auth, json=json_data)
+    
+    if response.status_code in [200, 201]:
+        return response.json() if response.content else True
+    elif response.status_code == 404 and method == "GET":
+        return None
+    else:
+        print(f"Error with {method} request: {response.status_code} - {response.text}")
+        return False
+    
+
+# Load YAML data for GUIDs to skip
+def load_disabled(disabled_file_path):
+    if os.path.exists(disabled_file_path):
+        with open(disabled_file_path, 'r') as file:
+            return yaml.safe_load(file) or {}
+    return {}
+
+def load_yaml_files(*dirs):
+    yaml_files = []
+    
+    for dir_path in dirs:
+        if os.path.isdir(dir_path):
+            # Recurse through the directory and subdirectories
+            for root, dirs, files in os.walk(dir_path):
+                for file_name in files:
+                    if file_name.endswith(".yaml"):
+                        full_path = os.path.join(root, file_name)
+                        with open(full_path, 'r') as f:
+                            try:
+                                yaml_content = yaml.safe_load(f)
+                                yaml_files.append(yaml_content)
+                            except yaml.YAMLError as e:
+                                print(f"Error loading {full_path}: {e}")
+        else:
+            print(f"Invalid directory: {dir_path}")
+    
+    return yaml_files
+
+def prepare_custom_rules(input_file, output_dir):
+    # Clear the output directory first
+    if os.path.exists(output_dir):
+        shutil.rmtree(output_dir)
+    os.makedirs(output_dir, exist_ok=True)
+    
+    try:
+        # Load the YAML file
+        with open(input_file, 'r') as f:
+            docs = yaml.safe_load_all(f)
+            
+            for doc in docs:
+                if 'id' not in doc:
+                    print(f"Skipping rule, no 'id' found: {doc}")
+                    continue
+                if doc.get('title') in ["Template 1", "Template 2"]:
+                    print(f"Skipping template rule with title: {doc['title']}")
+                    continue
+                # Create a filename using the 'id' field
+                file_name = os.path.join(output_dir, f"{doc['id']}.yaml")
+                
+                # Write the individual YAML file
+                with open(file_name, 'w') as output_file:
+                    yaml.dump(doc, output_file, default_flow_style=False)
+                print(f"Created file: {file_name}")
+    
+    except yaml.YAMLError as e:
+        print(f"Error parsing YAML: {e}")
+    except Exception as e:
+        print(f"Error processing file: {e}")

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -13,6 +13,9 @@
 
 LOG="/opt/so/log/elasticfleet/so-elastic-agent-gen-installers.log"
 
+# get the variables needed such as ELASTIC_AGENT_TARBALL_VERSION
+get_elastic_agent_vars
+
 # Check to see if we are already running
 NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers")
 [ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING gen installers script processes running...exiting." >>$LOG && exit 0
@@ -36,6 +39,7 @@ printf  "\n### Creating a temp directory at /nsm/elastic-agent-workspace\n"
 rm -rf /nsm/elastic-agent-workspace
 mkdir -p /nsm/elastic-agent-workspace
 
+
 printf "\n### Extracting outer tarball and then each individual tarball/zip\n"
 tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz -C /nsm/elastic-agent-workspace/
 unzip -q /nsm/elastic-agent-workspace/elastic-agent-*.zip -d /nsm/elastic-agent-workspace/
@@ -72,5 +76,17 @@ do
     printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done
 
-printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n"
+printf "\n\n### Generating MSI...\n"
+cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
+docker run \
+--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \
+{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
+printf "\n### MSI Generated...\n"
+
+printf "\n### Cleaning up temp files \n"
 rm -rf /nsm/elastic-agent-workspace
+rm -rf /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
+
+printf "\n### Copying so_agent-installers to /nsm/elastic-fleet/ for nginx.\n"
+\cp -vr /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/ /nsm/elastic-fleet/
+chmod 644 /nsm/elastic-fleet/so_agent-installers/*

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade

@@ -5,6 +5,7 @@
 # this file except in compliance with the Elastic License 2.0.
 
 . /usr/sbin/so-common
+{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 
 # Only run on Managers
 if ! is_manager_node; then
@@ -13,7 +14,7 @@ if ! is_manager_node; then
 fi
 
 # Get current list of Grid Node Agents that need to be upgraded
-RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=policy_id%20%3A%20so-grid-nodes_%2A&showInactive=false&showUpgradeable=true&getStatusSummary=true")
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/agents?perPage=20&page=1&kuery=NOT%20agent.version%20:%20%22{{ELASTICSEARCHDEFAULTS.elasticsearch.version}}%22%20and%20policy_id%20:%20%22so-grid-nodes_general%22&showInactive=false&getStatusSummary=true")
 
 # Check to make sure that the server responded with good data - else, bail from script
 CHECKSUM=$(jq -r '.page' <<< "$RAW_JSON")
@@ -27,14 +28,14 @@ OUTDATED_LIST=$(jq -r '.items | map(.id) | (tojson)'  <<<  "$RAW_JSON")
 
 if [ "$OUTDATED_LIST" != '[]' ]; then
    AGENTNUMBERS=$(jq -r '.total' <<< "$RAW_JSON")
-   printf "Initiating upgrades for $AGENTNUMBERS Agents to Elastic $ELASTIC_AGENT_TARBALL_VERSION...\n\n"
+   printf "Initiating upgrades for $AGENTNUMBERS Agents to Elastic {{ELASTICSEARCHDEFAULTS.elasticsearch.version}}...\n\n"
 
    # Generate updated JSON payload
-   JSON_STRING=$(jq -n --arg ELASTICVERSION $ELASTIC_AGENT_TARBALL_VERSION --arg UPDATELIST $OUTDATED_LIST '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')
+   JSON_STRING=$(jq -n --arg ELASTICVERSION {{ELASTICSEARCHDEFAULTS.elasticsearch.version}} --arg UPDATELIST $OUTDATED_LIST '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')
 
    # Update Node Agents
    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "http://localhost:5601/api/fleet/agents/bulk_upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 else
     printf "No Agents need updates... Exiting\n\n"
     exit 0
-fi
+fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-integration-upgrade

@@ -0,0 +1,73 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
+{%- set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %}
+
+. /usr/sbin/so-elastic-fleet-common
+
+curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/)
+if [ $? -ne 0 ]; then
+    echo "Error: Failed to connect to Kibana."
+    exit 1
+fi
+
+IFS=$'\n'
+agent_policies=$(elastic_fleet_agent_policy_ids)
+if [ $? -ne 0 ]; then
+    echo "Error: Failed to retrieve agent policies."
+    exit 1
+fi
+
+default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
+
+for AGENT_POLICY in $agent_policies; do
+    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+    for INTEGRATION in $integrations; do
+        if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
+            # Get package name so we know what package to look for when checking the current and latest available version
+            PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
+            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+            if [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
+            {%- endif %}
+                # Get currently installed version of package
+                PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION")
+
+                # Get latest available version of package
+                AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME")
+
+                # Get integration ID
+                INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION")
+
+                if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
+                    # Dry run of the upgrade
+                    echo ""
+                    echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
+                    echo "Upgrading $INTEGRATION..."
+                    echo "Starting dry run..."
+                    DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID")
+                    DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
+
+                    # If no errors with dry run, proceed with actual upgrade
+                    if [[ "$DRYRUN_ERRORS" == "false" ]]; then
+                        echo "No errors detected. Proceeding with upgrade..."
+                        elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
+                        if [ $? -ne 0 ]; then
+                            echo "Error: Upgrade failed for $PACKAGE_NAME with integration ID '$INTEGRATION_ID'."
+                            exit 1
+                        fi
+                    else
+                        echo "Errors detected during dry run for $PACKAGE_NAME policy upgrade..."
+                        exit 1
+                    fi
+                fi
+            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+            fi
+            {%- endif %}
+        fi
+    done
+done
+echo

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-optional-integrations-load

@@ -0,0 +1,169 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+# this file except in compliance with the Elastic License 2.0.
+{%- import_yaml 'elasticfleet/defaults.yaml' as ELASTICFLEETDEFAULTS %}
+{% set SUB = salt['pillar.get']('elasticfleet:config:subscription_integrations', default=false) %}
+{% set AUTO_UPGRADE_INTEGRATIONS = salt['pillar.get']('elasticfleet:config:auto_upgrade_integrations', default=false) %}
+{%- set SUPPORTED_PACKAGES = salt['pillar.get']('elasticfleet:packages', default=ELASTICFLEETDEFAULTS.elasticfleet.packages, merge=True) %}
+
+. /usr/sbin/so-common
+. /usr/sbin/so-elastic-fleet-common
+
+# Check that /opt/so/state/estemplates.txt exists to signal that Elasticsearch
+#  has completed its first run of core-only integrations/indices/components/ilm
+STATE_FILE_SUCCESS=/opt/so/state/estemplates.txt
+INSTALLED_PACKAGE_LIST=/tmp/esfleet_installed_packages.json
+BULK_INSTALL_PACKAGE_LIST=/tmp/esfleet_bulk_install.json
+BULK_INSTALL_PACKAGE_TMP=/tmp/esfleet_bulk_install_tmp.json
+BULK_INSTALL_OUTPUT=/opt/so/state/esfleet_bulk_install_results.json
+PACKAGE_COMPONENTS=/opt/so/state/esfleet_package_components.json
+
+PENDING_UPDATE=false
+
+# Integrations which are included in the package registry, but excluded from automatic installation via this script.
+#   Requiring some level of manual Elastic Stack configuration before installation
+EXCLUDED_INTEGRATIONS=('apm')
+
+version_conversion(){
+    version=$1
+    echo "$version" | awk -F '.' '{ printf("%d%03d%03d\n", $1, $2, $3); }'
+}
+
+compare_versions() {
+    version1=$1
+    version2=$2
+
+    # Convert versions to numbers
+    num1=$(version_conversion "$version1")
+    num2=$(version_conversion "$version2")
+
+    # Compare using bc
+    if (( $(echo "$num1 < $num2" | bc -l) )); then
+        echo "less"
+    elif (( $(echo "$num1 > $num2" | bc -l) )); then
+        echo "greater"
+    else
+        echo "equal"
+    fi
+}
+
+IFS=$'\n'
+agent_policies=$(elastic_fleet_agent_policy_ids)
+if [ $? -ne 0 ]; then
+    echo "Error: Failed to retrieve agent policies."
+    exit 1
+fi
+
+default_packages=({% for pkg in SUPPORTED_PACKAGES %}"{{ pkg }}"{% if not loop.last %} {% endif %}{% endfor %})
+
+in_use_integrations=()
+
+for AGENT_POLICY in $agent_policies; do
+    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
+    for INTEGRATION in $integrations; do
+        PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
+        # non-default integrations that are in-use in any policy
+        if ! [[ " ${default_packages[@]} " =~ " $PACKAGE_NAME " ]]; then
+            in_use_integrations+=("$PACKAGE_NAME")
+        fi
+    done
+done
+
+if [[ -f $STATE_FILE_SUCCESS  ]]; then
+    if retry 3 1 "curl -s -K /opt/so/conf/elasticsearch/curl.config --output /dev/null --silent --head --fail localhost:5601/api/fleet/epm/packages"; then
+        # Package_list contains all integrations beta / non-beta.
+        latest_package_list=$(/usr/sbin/so-elastic-fleet-package-list)
+        echo '{ "packages" : []}' > $BULK_INSTALL_PACKAGE_LIST
+        rm -f $INSTALLED_PACKAGE_LIST
+        echo $latest_package_list | jq '{packages: [.items[] | {name: .name, latest_version: .version, installed_version: .savedObject.attributes.install_version, subscription: .conditions.elastic.subscription }]}' >> $INSTALLED_PACKAGE_LIST
+
+        while read -r package; do
+            # get package details
+            package_name=$(echo "$package" | jq -r '.name')
+            latest_version=$(echo "$package" | jq -r '.latest_version')
+            installed_version=$(echo "$package" | jq -r '.installed_version')
+            subscription=$(echo "$package" | jq -r '.subscription')
+            bulk_package=$(echo "$package" | jq '{name: .name, version: .latest_version}' )
+
+            if [[ ! "${EXCLUDED_INTEGRATIONS[@]}" =~ "$package_name" ]]; then
+            {% if not SUB %}
+                if [[ "$subscription" != "basic" && "$subscription" != "null" && -n "$subscription" ]]; then
+                    # pass over integrations that require non-basic elastic license
+                    echo "$package_name integration requires an Elastic license of $subscription or greater... skipping"
+                    continue
+                else
+                    if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then
+                        echo "$package_name is not installed... Adding to next update."
+                        jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+
+                        PENDING_UPDATE=true
+                    else
+                        results=$(compare_versions "$latest_version" "$installed_version")
+                        if [ $results == "greater" ]; then
+                            {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations  #}
+                            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                            if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
+                            {%- endif %}
+                                echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
+                                jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+
+                                PENDING_UPDATE=true
+                            {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                            else
+                                echo "skipping available upgrade for in use integration - $package_name."
+                            fi
+                            {%- endif %}
+                        fi
+                    fi
+                fi
+            {% else %}
+                if [[ "$installed_version" == "null" || -z "$installed_version" ]]; then
+                    echo "$package_name is not installed... Adding to next update."
+                    jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                    PENDING_UPDATE=true
+                else
+                    results=$(compare_versions "$latest_version" "$installed_version")
+                    if [ $results == "greater" ]; then
+                        {#- When auto_upgrade_integrations is false, skip upgrading in_use_integrations  #}
+                        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                        if ! [[ " ${in_use_integrations[@]} " =~ " $package_name " ]]; then
+                        {%- endif %}
+                            echo "$package_name is at version $installed_version latest version is $latest_version... Adding to next update."
+                            jq --argjson package "$bulk_package" '.packages += [$package]' $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_PACKAGE_TMP && mv $BULK_INSTALL_PACKAGE_TMP $BULK_INSTALL_PACKAGE_LIST
+                            PENDING_UPDATE=true
+                        {%- if not AUTO_UPGRADE_INTEGRATIONS %}
+                        else
+                            echo "skipping available upgrade for in use integration - $package_name."
+                        fi
+                        {%- endif %}
+                    fi
+                fi
+            {% endif %}
+            else
+                echo "Skipping $package_name..."
+            fi
+        done <<< "$(jq -c '.packages[]' "$INSTALLED_PACKAGE_LIST")"
+
+        if [ "$PENDING_UPDATE" = true ]; then
+            # Run bulk install of packages
+            elastic_fleet_bulk_package_install $BULK_INSTALL_PACKAGE_LIST > $BULK_INSTALL_OUTPUT
+        else
+            echo "Elastic integrations don't appear to need installation/updating..."
+        fi
+        # Write out file for generating index/component/ilm templates
+        latest_installed_package_list=$(elastic_fleet_installed_packages)
+        echo $latest_installed_package_list | jq '[.items[] | {name: .name, es_index_patterns: .dataStreams}]' > $PACKAGE_COMPONENTS
+
+    else
+        # This is the installation of add-on integrations and upgrade of existing integrations. Exiting without error, next highstate will attempt to re-run.
+        echo "Elastic Fleet does not appear to be responding... Exiting... "
+        exit 0
+    fi
+else
+    # This message will appear when an update to core integration is made and this script is run at the same time as
+    #  elasticsearch.enabled -> detects change to core index settings -> deletes estemplates.txt
+    echo "Elasticsearch may not be fully configured yet or is currently updating core index settings."
+    exit 0
+fi

salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy

@@ -32,7 +32,7 @@ if ! echo "$output" | grep -q "so-manager_kafka"; then
                 --arg KAFKACA "$KAFKACA" \
                 --arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \
                 --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
-                    '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-securityonion","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
+                    '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 10 }, "topics":[{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
                 )
   curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" -o /dev/null
   refresh_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)

salt/elasticsearch/auth.sls

@@ -15,7 +15,7 @@
 elastic_auth_pillar:
   file.managed:
     - name: /opt/so/saltstack/local/pillar/elasticsearch/auth.sls
-    - mode: 600
+    - mode: 640
     - reload_pillar: True
     - contents: |
         elasticsearch:

salt/elasticsearch/config.sls

@@ -47,6 +47,7 @@ elasticsearch_sbin:
     - file_mode: 755
     - exclude_pat:
       - so-elasticsearch-pipelines # exclude this because we need to watch it for changes, we sync it in another state
+    - show_changes: False
 
 elasticsearch_sbin_jinja:
   file.recurse:
@@ -60,6 +61,7 @@ elasticsearch_sbin_jinja:
       - so-elasticsearch-ilm-policy-load # exclude this because we need to watch it for changes, we sync it in another state
     - defaults:
         GLOBALS: {{ GLOBALS }}
+    - show_changes: False
 
 so-elasticsearch-ilm-policy-load-script:
   file.managed:
@@ -69,6 +71,7 @@ so-elasticsearch-ilm-policy-load-script:
     - group: 939
     - mode: 754
     - template: jinja
+    - show_changes: False
 
 so-elasticsearch-pipelines-script:
   file.managed:
@@ -77,6 +80,7 @@ so-elasticsearch-pipelines-script:
     - user: 930
     - group: 939
     - mode: 754
+    - show_changes: False
 
 esingestdir:
   file.directory:
@@ -110,6 +114,7 @@ esingestdynamicconf:
     - user: 930
     - group: 939
     - template: jinja
+    - show_changes: False
 
 esingestconf:
   file.recurse:
@@ -117,6 +122,7 @@ esingestconf:
     - source: salt://elasticsearch/files/ingest
     - user: 930
     - group: 939
+    - show_changes: False
 
 # Remove .fleet_final_pipeline-1 because we are using global@custom now
 so-fleet-final-pipeline-remove:
@@ -153,6 +159,7 @@ esyml:
     - defaults:
         ESCONFIG: {{ ELASTICSEARCHMERGED.config }}
     - template: jinja
+    - show_changes: False
 
 esroles:
   file.recurse:
@@ -162,6 +169,7 @@ esroles:
     - template: jinja
     - user: 930
     - group: 939
+    - show_changes: False
 
 nsmesdir:
   file.directory:

salt/elasticsearch/download.sls

@@ -6,10 +6,11 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
 
 so-elasticsearch_image:
   docker_image.present:
-    - name: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ GLOBALS.so_version }}
+    - name: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}
 
 {% else %}
 

salt/elasticsearch/enabled.sls

@@ -19,7 +19,7 @@ include:
 
 so-elasticsearch:
   docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ GLOBALS.so_version }}
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ ELASTICSEARCHMERGED.version }}
     - hostname: elasticsearch
     - name: so-elasticsearch
     - user: elasticsearch
@@ -38,7 +38,7 @@ so-elasticsearch:
       {% endfor %}
     {% endif %}
     - environment:
-      {% if ELASTICSEARCH_SEED_HOSTS | length == 1 or GLOBALS.role == 'so-heavynode' %}
+      {% if (GLOBALS.role in GLOBALS.manager_roles and ELASTICSEARCH_SEED_HOSTS | length == 1) or GLOBALS.role == 'so-heavynode' %}
       - discovery.type=single-node
       {% endif %}
       - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true
@@ -116,6 +116,7 @@ escomponenttemplates:
     - clean: True
     - onchanges_in:
       - file: so-elasticsearch-templates-reload
+    - show_changes: False
       
 # Auto-generate templates from defaults file
 {%     for index, settings in ES_INDEX_SETTINGS.items() %}
@@ -127,6 +128,7 @@ es_index_template_{{index}}:
     - defaults:
       TEMPLATE_CONFIG: {{ settings.index_template }}
     - template: jinja
+    - show_changes: False
     - onchanges_in:
       - file: so-elasticsearch-templates-reload
 {%       endif %}
@@ -146,12 +148,13 @@ es_template_{{TEMPLATE.split('.')[0] | replace("/","_") }}:
 {%         endif %}
     - user: 930
     - group: 939
+    - show_changes: False
     - onchanges_in:
       - file: so-elasticsearch-templates-reload
 {%       endfor %}
 {%     endif %}
 
-{% if GLOBALS.role in GLOBALS.manager_roles %}
+{%     if GLOBALS.role in GLOBALS.manager_roles %}
 so-es-cluster-settings:
   cmd.run:
     - name: /usr/sbin/so-elasticsearch-cluster-settings
@@ -160,7 +163,7 @@ so-es-cluster-settings:
     - require:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
-{% endif %}
+{%     endif %}
 
 so-elasticsearch-ilm-policy-load:
   cmd.run:
@@ -201,12 +204,17 @@ so-elasticsearch-roles-load:
       - docker_container: so-elasticsearch
       - file: elasticsearch_sbin_jinja
 
-{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
+{%     if grains.role in ['so-managersearch', 'so-manager'] %}
+{%       set ap = "absent" %}
+{%     endif %}
+{%     if grains.role in ['so-eval', 'so-standalone', 'so-heavynode'] %}
 {%       if ELASTICSEARCHMERGED.index_clean %}
 {%         set ap = "present" %}
 {%       else %}
 {%         set ap = "absent" %}
 {%       endif %}
+{%     endif %}  
+{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
 so-elasticsearch-indices-delete:
   cron.{{ap}}:
     - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1

salt/elasticsearch/files/ingest/global@custom

@@ -8,20 +8,22 @@
   "processors": [
     { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
     { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
+    { "split":      { "if": "ctx.data_stream?.dataset != null && ctx.data_stream?.dataset.contains('.')", "field":"data_stream.dataset", "separator":"\\.", "target_field":"datastream_dataset_temp", "ignore_missing":true } },
     { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
+    { "set":        { "if": "ctx.datastream_dataset_temp != null && ctx.datastream_dataset_temp[0] == 'network_traffic'", "field":"event.module", "value":"{{ datastream_dataset_temp.0 }}", "ignore_failure":true, "ignore_empty_value":true, "description":"Fix EA network packet capture" } },
     { "gsub":       { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
-    { "append":     { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}"  } },
+    { "append":     { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}", "allow_duplicates": false  } },
     { "set":        { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
     { "set":        { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
     { "set":        { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
     { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
-    { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } },
-    { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } },
+    { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.dataset", "value": "import" } },
+    { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.namespace", "value": "so" } },
     { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
     { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
     { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
     { "rename":        { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
     { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
-    { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } 
+    { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp", "datastream_dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
   ]
 }

salt/elasticsearch/files/ingest/hydra

@@ -0,0 +1,9 @@
+{
+  "description" : "hydra",
+  "processors" : [
+    {"set":{"field":"audience","value":"access","override":false,"ignore_failure":true}},
+    {"set":{"field":"event.dataset","ignore_empty_value":true,"ignore_failure":true,"value":"hydra.{{{audience}}}","media_type":"text/plain"}},
+    {"set":{"field":"event.action","ignore_failure":true,"copy_from":"msg"    }},
+    { "pipeline":    { "name": "common" } }
+  ]
+}

salt/elasticsearch/files/ingest/logs-pfsense.log-1.21.0

@@ -1,10 +1,17 @@
 {
-  "description": "Pipeline for pfSense",
+  "description": "Pipeline for PFsense",
+  "_meta": {
+    "managed_by": "fleet",
+    "managed": true,
+    "package": {
+      "name": "pfsense"
+    }
+  },
   "processors": [
     {
       "set": {
         "field": "ecs.version",
-        "value": "8.10.0"
+        "value": "8.17.0"
       }
     },
     {
@@ -22,7 +29,9 @@
     {
       "rename": {
         "field": "message",
-        "target_field": "event.original"
+        "target_field": "event.original",
+        "ignore_missing": true,
+        "if": "ctx.event?.original == null"
       }
     },
     {
@@ -34,7 +43,7 @@
     {
       "set": {
         "field": "event.timezone",
-        "value": "{{_tmp.tz_offset}}",
+        "value": "{{{_tmp.tz_offset}}}",
         "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'"
       }
     },
@@ -54,7 +63,8 @@
           "SYSLOG_TIMESTAMP_FORMAT": "%{TIMESTAMP_ISO8601:_tmp.timestamp8601}%{SPACE}%{OBSERVER}%{SPACE}%{PROCESS}%{SPACE}(%{POSINT:process.pid:long}|-) - (-|%{META})",
           "TIMESTAMP_ISO8601": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE:event.timezone}?",
           "OBSERVER": "(?:%{IP:observer.ip}|%{HOSTNAME:observer.name})",
-          "PROCESS": "(\\(%{DATA:process.name}\\)|(?:%{UNIXPATH}*/)?%{BASEPATH:process.name})",
+          "UNIXPATH": "(/([\\w_%!$@:.,+~-]+|\\\\.)*)*",
+          "PROCESS": "(\\(%{DATA:process.name}\\)|(?:%{UNIXPATH})%{BASEPATH:process.name})",
           "BASEPATH": "[[[:alnum:]]_%!$@:.,+~-]+",
           "META": "\\[[^\\]]*\\]"
         }
@@ -80,7 +90,7 @@
           "MMM d HH:mm:ss",
           "MMM dd HH:mm:ss"
         ],
-        "timezone": "{{ event.timezone }}"
+        "timezone": "{{{ event.timezone }}}"
       }
     },
     {
@@ -97,61 +107,67 @@
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-firewall",
+        "name": "logs-pfsense.log-1.21.0-firewall",
         "if": "ctx.event.provider == 'filterlog'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-openvpn",
+        "name": "logs-pfsense.log-1.21.0-openvpn",
         "if": "ctx.event.provider == 'openvpn'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-ipsec",
+        "name": "logs-pfsense.log-1.21.0-ipsec",
         "if": "ctx.event.provider == 'charon'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-dhcp",
+        "name": "logs-pfsense.log-1.21.0-dhcp",
         "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-unbound",
+        "name": "logs-pfsense.log-1.21.0-unbound",
         "if": "ctx.event.provider == 'unbound'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-haproxy",
+        "name": "logs-pfsense.log-1.21.0-haproxy",
         "if": "ctx.event.provider == 'haproxy'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-php-fpm",
+        "name": "logs-pfsense.log-1.21.0-php-fpm",
         "if": "ctx.event.provider == 'php-fpm'"
       }
     },
     {
       "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-squid",
+        "name": "logs-pfsense.log-1.21.0-squid",
         "if": "ctx.event.provider == 'squid'"
       }
     },
     {
-     "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-suricata",
+      "pipeline": {
+        "name": "logs-pfsense.log-1.21.0-snort",
+        "if": "ctx.event.provider == 'snort'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.21.0-suricata",
         "if": "ctx.event.provider == 'suricata'"
       }
     },
     {
       "drop": {
-        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"suricata\"].contains(ctx.event?.provider)"
+        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"snort\", \"suricata\"].contains(ctx.event?.provider)"
       }
     },
     {
@@ -285,7 +301,7 @@
     {
       "append": {
         "field": "related.ip",
-        "value": "{{destination.ip}}",
+        "value": "{{{destination.ip}}}",
         "allow_duplicates": false,
         "if": "ctx.destination?.ip != null"
       }
@@ -293,7 +309,7 @@
     {
       "append": {
         "field": "related.ip",
-        "value": "{{source.ip}}",
+        "value": "{{{source.ip}}}",
         "allow_duplicates": false,
         "if": "ctx.source?.ip != null"
       }
@@ -301,7 +317,7 @@
     {
       "append": {
         "field": "related.ip",
-        "value": "{{source.nat.ip}}",
+        "value": "{{{source.nat.ip}}}",
         "allow_duplicates": false,
         "if": "ctx.source?.nat?.ip != null"
       }
@@ -309,21 +325,21 @@
     {
       "append": {
         "field": "related.hosts",
-        "value": "{{destination.domain}}",
+        "value": "{{{destination.domain}}}",
         "if": "ctx.destination?.domain != null"
       }
     },
     {
       "append": {
         "field": "related.user",
-        "value": "{{user.name}}",
+        "value": "{{{user.name}}}",
         "if": "ctx.user?.name != null"
       }
     },
     {
       "set": {
         "field": "network.direction",
-        "value": "{{network.direction}}bound",
+        "value": "{{{network.direction}}}bound",
         "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/"
       }
     },
@@ -350,10 +366,32 @@
         "ignore_missing": true
       }
     },
+    {
+      "pipeline": {
+        "name": "global@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Global pipeline for all data streams"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for all data streams of type `logs`"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.integration@custom",
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `pfsense` integration"
+      }
+    },
     {
       "pipeline": {
         "name": "logs-pfsense.log@custom",
-        "ignore_missing_pipeline": true
+        "ignore_missing_pipeline": true,
+        "description": "[Fleet] Pipeline for the `pfsense.log` dataset"
       }
     }
   ],
@@ -378,12 +416,5 @@
         "value": "{{{ _ingest.on_failure_message }}}"
       }
     }
-  ],
-  "_meta": {
-    "managed_by": "fleet",
-    "managed": true,
-    "package": {
-      "name": "pfsense"
-    }
-  }
-}
+  ]
+}

salt/elasticsearch/files/ingest/logs-pfsense.log-1.21.0-suricata

@@ -1,9 +1,14 @@
 {
   "description": "Pipeline for parsing pfSense Suricata logs.",
   "processors": [
+    { "set": {
+        "field": "event.module",
+        "value": "suricata"
+      }
+    },
     {
      "pipeline": {
-        "name": "suricata.common"
+        "name": "suricata.common_pfsense"
       }
     }
   ],

salt/elasticsearch/files/ingest/suricata.alert

@@ -1,7 +1,7 @@
 {
   "description" : "suricata.alert",
   "processors" : [
-    { "set":   { "field": "_index", "value": "logs-suricata.alerts-so" } },
+    { "set":   { "if": "ctx.event?.imported != true", "field": "_index", "value": "logs-suricata.alerts-so" } },
     { "set":   { "field": "tags","value": "alert" }},
     { "rename":{ "field": "message2.alert",	"target_field": "rule",	"ignore_failure": true 	} },
     { "rename":{ "field": "rule.signature",     "target_field": "rule.name", "ignore_failure": true  } },

salt/elasticsearch/files/ingest/suricata.alert_pfsense

@@ -0,0 +1,16 @@
+{
+  "description" : "suricata.alert",
+  "processors" : [
+    { "set":   { "field": "data_stream.dataset", "value": "suricata" } },
+    { "set":   { "field": "data_stream.namespace", "value": "so" } },
+    { "set":   { "field": "_index", "value": "logs-suricata.alerts-so" } },
+    { "set":   { "field": "tags","value": "alert" }},
+    { "rename":{ "field": "message2.alert",	"target_field": "rule",	"ignore_failure": true 	} },
+    { "rename":{ "field": "rule.signature",     "target_field": "rule.name", "ignore_failure": true  } },
+    { "rename":{ "field": "rule.ref",     "target_field": "rule.version", "ignore_failure": true  } },
+    { "rename":{ "field": "rule.signature_id",     "target_field": "rule.uuid", "ignore_failure": true  } },
+    { "rename":{ "field": "rule.signature_id",     "target_field": "rule.signature", "ignore_failure": true  } },
+    { "rename":{ "field": "message2.payload_printable",     "target_field": "network.data.decoded", "ignore_failure": true  } },
+    { "pipeline": { "name": "common.nids" } }
+  ]
+}

salt/elasticsearch/files/ingest/suricata.common_pfsense

@@ -0,0 +1,23 @@
+{
+  "description" : "suricata.common",
+  "processors" : [
+    { "json":   { "field": "message",                   "target_field": "message2",             "ignore_failure": true  } },
+    { "rename": { "field": "message2.pkt_src",          "target_field": "network.packet_source","ignore_failure": true  } },
+    { "rename": { "field": "message2.proto",            "target_field": "network.transport",    "ignore_failure": true  } },
+    { "rename": { "field": "message2.in_iface",         "target_field": "observer.ingress.interface.name",    "ignore_failure": true  } },
+    { "rename": { "field": "message2.flow_id",          "target_field": "log.id.uid",           "ignore_failure": true  } },
+    { "rename": { "field": "message2.src_ip",           "target_field": "source.ip",            "ignore_failure": true  } },
+    { "rename": { "field": "message2.src_port",         "target_field": "source.port",          "ignore_failure": true  } },
+    { "rename": { "field": "message2.dest_ip",          "target_field": "destination.ip",       "ignore_failure": true  } },
+    { "rename": { "field": "message2.dest_port",        "target_field": "destination.port",     "ignore_failure": true  } },
+    { "rename": { "field": "message2.vlan",             "target_field": "network.vlan.id",      "ignore_failure": true  } },
+    { "rename": { "field": "message2.community_id",     "target_field": "network.community_id", "ignore_missing": true  } },
+    { "rename": { "field": "message2.xff",              "target_field": "xff.ip",               "ignore_missing": true  } },
+    { "set":    { "field": "event.dataset",             "value": "{{ message2.event_type }}"                            } },
+    { "set":    { "field": "observer.name",             "value": "{{agent.name}}"                                       } },
+    { "set":    { "field": "event.ingested",            "value": "{{@timestamp}}"                                       } },
+    { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
+    { "remove":{ "field": "agent",                                                              "ignore_failure": true  } },
+    { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}_pfsense" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.common

@@ -18,6 +18,7 @@
     { "set":         { "if": "ctx.destination?.ip != null", "field": "server.ip",        "value": "{{destination.ip}}"  } },
     { "set":         { "if": "ctx.destination?.port != null", "field": "server.port",        "value": "{{destination.port}}"  } },
     { "set":         { "field": "observer.name",        "value": "{{agent.name}}"  } },
+    { "append": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"openvpn\")","field": "tags","value": ["{{network.protocol}}"],"allow_duplicates": false,"ignore_failure": true}},
     { "date":		{ "field": "message2.ts",	"target_field": "@timestamp",	"formats": ["ISO8601", "UNIX"], "ignore_failure": true	} },
     { "remove":		{ "field": ["agent"],	"ignore_failure": true									} },
     { "pipeline":	{ "name": "common" 													} }

salt/elasticsearch/files/ingest/zeek.conn

@@ -38,6 +38,8 @@
     { "set": { "if": "ctx.connection?.state == 'SH'",    "field": "connection.state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } },
     { "set": { "if": "ctx.connection?.state == 'SHR'",   "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } },
     { "set": { "if": "ctx.connection?.state == 'OTH'",   "field": "connection.state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } },
+    { "set": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"ipsec\")", "field": "network.protocol", "value": "ipsec"}},
+    { "set": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"openvpn\")", "field": "network.protocol", "value": "openvpn"}},
     { "pipeline": 	{ "name": "zeek.common" } }
   ]
 }

salt/elasticsearch/files/ingest/zeek.http2

@@ -0,0 +1,37 @@
+{
+  "description" : "zeek.http2",
+  "processors" : [
+    { "set":      { "field": "event.dataset",                   "value": "http2" } },
+    { "set":      { "field": "network.transport",               "value": "tcp"  } },
+    { "json":     { "field": "message",                         "target_field": "message2",                       "ignore_failure": true  } },
+    { "rename": 	{ "field": "message2.trans_depth",	          "target_field": "http.trans_depth",		        "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.method", 		            "target_field": "http.method",		            "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.host", 		              "target_field": "http.virtual_host",		      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.uri", 		                "target_field": "http.uri",			              "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.referrer", 	            "target_field": "http.referrer",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.version", 		            "target_field": "http.version",		            "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.user_agent", 	          "target_field": "http.useragent",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.request_body_len",       "target_field": "http.request.body.length",	  "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.response_body_len",      "target_field": "http.response.body.length",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.status_code",	          "target_field": "http.status_code",		        "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.status_msg", 	          "target_field": "http.status_message",	      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.info_code", 	            "target_field": "http.info_code",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.info_msg", 	            "target_field": "http.info_message",		      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.username", 	            "target_field": "http.user",			            "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.password", 	            "target_field": "http.password",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.proxied", 		            "target_field": "http.proxied",		            "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.orig_fuids",	            "target_field": "log.id.orig_fuids",		      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.orig_filenames",	        "target_field": "file.orig_filenames",	      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.orig_mime_types",	      "target_field": "file.orig_mime_types",	      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.resp_fuids",	            "target_field": "log.id.resp_fuids",		      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.resp_filenames",	        "target_field": "file.resp_filenames",	      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.resp_mime_types",	      "target_field": "file.resp_mime_types",	      "ignore_missing": true 	} },
+    { "rename":   { "field": "message2.stream_id",              "target_field": "http2.stream_id",            "ignore_missing": true  } },
+    { "remove":		{ "field": "message2.tags",		                                                                  "ignore_failure": true } },
+    { "remove":   { "field": ["host"],                                                                            "ignore_failure": true } },
+    { "script":	{ "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", 			                        "ignore_failure": true 	} },
+    { "script":	{ "lang": "painless", "source": "ctx.useragent_length = ctx.useragent.length()",	                "ignore_failure": true 	} },
+    { "script":	{ "lang": "painless", "source": "ctx.virtual_host_length = ctx.virtual_host.length()", 	          "ignore_failure": true	} },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.ipsec

@@ -0,0 +1,38 @@
+{
+  "description": "zeek.ipsec",
+  "processors": [
+    {"set": { "field": "event.dataset","value": "ipsec"}},
+    {"json": { "field": "message","target_field": "message2","ignore_failure": true}},
+    {"rename": {"field": "message2.initiator_spi","target_field": "ipsec.initiator_spi","ignore_missing": true}},
+    {"rename": {"field": "message2.responder_spi","target_field": "ipsec.responder_spi","ignore_missing": true}},
+    {"rename": {"field": "message2.maj_ver","target_field": "ipsec.maj_version","ignore_missing": true}},
+    {"rename": {"field": "message2.min_ver","target_field": "ipsec.min_version","ignore_missing": true}},
+    {"set": {"ignore_failure": true,"field": "ipsec.version","value": "{{ipsec.maj_version}}.{{ipsec.min_version}}"}},
+    {"rename": {"field": "message2.exchange_type","target_field": "ipsec.exchange_type","ignore_missing": true}},
+    {"rename": {"field": "message2.flag_e","target_field": "ipsec.flag_e","ignore_missing": true}},
+    {"rename": {"field": "message2.flag_c","target_field": "ipsec.flag_c","ignore_missing": true}},
+    {"rename": {"field": "message2.flag_a","target_field": "ipsec.flag_a","ignore_missing": true}},
+    {"rename": {"field": "message2.flag_i","target_field": "ipsec.flag_i","ignore_missing": true}},
+    {"rename": {"field": "message2.flag_v","target_field": "ipsec.flag_v","ignore_missing": true}},
+    {"rename": {"field": "message2.flag_r","target_field": "ipsec.flag_r","ignore_missing": true}},
+    {"rename": {"field": "message2.message_id","target_field": "ipsec.message_id","ignore_missing": true}},
+    {"rename": {"field": "message2.vendor_ids","target_field": "ipsec.vendor_ids","ignore_missing": true}},
+    {"rename": {"field": "message2.notify_messages","target_field": "ipsec.notify_messages","ignore_missing": true}},
+    {"rename": {"field": "message2.transforms","target_field": "ipsec.transforms","ignore_missing": true}},
+    {"rename": {"field": "message2.ke_dh_groups","target_field": "ipsec.ke_dh_groups","ignore_missing": true}},
+    {"rename": {"field": "message2.proposals","target_field": "ipsec.proposals","ignore_missing": true}},
+    {"rename": {"field": "message2.certificates","target_field": "ipsec.certificates","ignore_missing": true}},
+    {"rename": {"field": "message2.transform_attributes","target_field": "ipsec.transform_attributes","ignore_missing": true}},
+    {"rename": {"field": "message2.length","target_field": "ipsec.length","ignore_missing": true}},
+    {"rename": {"field": "message2.hash","target_field": "ipsec.hash","ignore_missing": true}},
+    {"rename": {"field": "message2.doi","target_field": "ipsec.doi","ignore_missing": true}},
+    {"rename": {"field": "message2.situation","target_field": "ipsec.situation","ignore_missing": true}},
+    {"script": {
+      "lang": "painless",
+      "description": "Remove ipsec fields with empty arrays",
+      "source": "if (ctx.containsKey('ipsec') && ctx.ipsec instanceof Map) {\n        for (String field : ['certificates', 'ke_dh_groups', 'notify_messages', 'proposals', 'transforms', 'transform_attributes', 'vendor_ids']) {\n          if (ctx.ipsec[field] instanceof List && ctx.ipsec[field].isEmpty()) {\n            ctx.ipsec.remove(field);\n          }\n        }\n      }",
+      "ignore_failure": true
+      }},
+    {"pipeline": {"name": "zeek.common"}}
+  ]
+}

salt/elasticsearch/files/ingest/zeek.ldap

@@ -0,0 +1,25 @@
+{
+    "description": "zeek.ldap",
+    "processors": [
+        {"set":      {"field": "event.dataset",                 "value": "ldap"}},
+        {"json":     {"field": "message",                       "target_field": "message2",                     "ignore_failure": true}},
+        {"rename":   {"field": "message2.message_id",           "target_field": "ldap.message_id",              "ignore_missing": true}},
+        {"rename":   {"field": "message2.opcode",               "target_field": "ldap.opcode",                  "ignore_missing": true}},
+        {"rename":   {"field": "message2.result",               "target_field": "ldap.result",                  "ignore_missing": true}},
+        {"rename":   {"field": "message2.diagnostic_message",   "target_field": "ldap.diagnostic_message",      "ignore_missing": true}},
+        {"rename":   {"field": "message2.version",              "target_field": "ldap.version",                 "ignore_missing": true}},
+        {"rename":   {"field": "message2.object",               "target_field": "ldap.object",                  "ignore_missing": true}},
+        {"rename":   {"field": "message2.argument",             "target_field": "ldap.argument",                "ignore_missing": true}},
+        {"rename":   {"field": "message2.scope",                "target_field": "ldap_search.scope",            "ignore_missing":true}},
+        {"rename":   {"field": "message2.deref_aliases",        "target_field": "ldap_search.deref_aliases",    "ignore_missing":true}},
+        {"rename":   {"field": "message2.base_object",          "target_field": "ldap.object",                  "ignore_missing":true}},
+        {"rename":   {"field": "message2.result_count",         "target_field": "ldap_search.result_count",     "ignore_missing":true}},
+        {"rename":   {"field": "message2.filter",               "target_field": "ldap_search.filter",           "ignore_missing":true}},
+        {"rename":   {"field": "message2.attributes",           "target_field": "ldap_search.attributes",       "ignore_missing":true}},
+        {"script":   {"source": "if (ctx.containsKey('ldap') && ctx.ldap.containsKey('diagnostic_message') && ctx.ldap.diagnostic_message != null) {\n          String message = ctx.ldap.diagnostic_message;\n\n          // get user and property from SASL success\n          if (message.toLowerCase().contains(\"sasl(0): successful result\")) {\n            Pattern pattern = /user:\\s*([^ ]+)\\s*property:\\s*([^ ]+)/i;\n            Matcher matcher = pattern.matcher(message);\n            if (matcher.find()) {\n              ctx.ldap.user_email = matcher.group(1);  // Extract user email\n              ctx.ldap.property = matcher.group(2);    // Extract property\n            }\n          }\n          if (message.toLowerCase().contains(\"ldaperr:\")) {\n            Pattern pattern = /comment:\\s*([^,]+)/i;\n            Matcher matcher = pattern.matcher(message);\n\n            if (matcher.find()) {\n              ctx.ldap.comment = matcher.group(1);\n            }\n          }\n        }","ignore_failure": true}},
+        {"script":   {"source": "if (ctx.containsKey('ldap') && ctx.ldap.containsKey('object') && ctx.ldap.object != null) {\n    String message = ctx.ldap.object;\n\n    // parse common name from ldap object\n    if (message.toLowerCase().contains(\"cn=\")) {\n        Pattern pattern = /cn=([^,]+)/i;\n        Matcher matcher = pattern.matcher(message);\n        if (matcher.find()) {\n            ctx.ldap.common_name = matcher.group(1);  // Extract CN\n        }\n    }\n    // build domain from ldap object\n    if (message.toLowerCase().contains(\"dc=\")) {\n        Pattern dcPattern = /dc=([^,]+)/i;\n        Matcher dcMatcher = dcPattern.matcher(message);\n\n        StringBuilder domainBuilder = new StringBuilder();\n        while (dcMatcher.find()) {\n            if (domainBuilder.length() > 0 ){\n                domainBuilder.append(\".\");\n            }\n            domainBuilder.append(dcMatcher.group(1));\n        }\n        if (domainBuilder.length() > 0) {\n            ctx.ldap.domain = domainBuilder.toString();\n        }\n    }\n    // create list of any organizational units from ldap object\n    if (message.toLowerCase().contains(\"ou=\")) {\n        Pattern ouPattern = /ou=([^,]+)/i;\n        Matcher ouMatcher = ouPattern.matcher(message);\n        ctx.ldap.organizational_unit = [];\n\n        while (ouMatcher.find()) {\n            ctx.ldap.organizational_unit.add(ouMatcher.group(1));\n        }\n        if(ctx.ldap.organizational_unit.isEmpty()) {\n          ctx.remove(\"ldap.organizational_unit\");\n        }\n    }\n}\n","ignore_failure": true}},
+        {"remove":   {"field": "message2.tags","ignore_failure": true}},
+        {"remove":   {"field": ["host"],"ignore_failure": true}},
+        {"pipeline": {"name": "zeek.common"}}
+    ]
+}

salt/elasticsearch/files/ingest/zeek.ldap_search

@@ -0,0 +1,25 @@
+{
+    "description":"zeek.ldap_search",
+    "processors":[
+        {"set":         {"field": "event.dataset",                 "value":"ldap_search"}},
+        {"json":        {"field": "message",                       "target_field": "message2",                     "ignore_failure": true}},
+        {"rename":      {"field": "message2.message_id",           "target_field": "ldap.message_id",              "ignore_missing": true}},
+        {"rename":      {"field": "message2.opcode",               "target_field": "ldap.opcode",                  "ignore_missing": true}},
+        {"rename":      {"field": "message2.result",               "target_field": "ldap.result",                  "ignore_missing": true}},
+        {"rename":      {"field": "message2.diagnostic_message",   "target_field": "ldap.diagnostic_message",      "ignore_missing": true}},
+        {"rename":      {"field": "message2.version",              "target_field": "ldap.version",                 "ignore_missing": true}},
+        {"rename":      {"field": "message2.object",               "target_field": "ldap.object",                  "ignore_missing": true}},
+        {"rename":      {"field": "message2.argument",             "target_field": "ldap.argument",                "ignore_missing": true}},
+        {"rename":      {"field": "message2.scope",                "target_field": "ldap_search.scope",            "ignore_missing":true}},
+        {"rename":      {"field": "message2.deref_aliases",        "target_field": "ldap_search.deref_aliases",    "ignore_missing":true}},
+        {"rename":      {"field": "message2.base_object",          "target_field": "ldap.object",                  "ignore_missing":true}},
+        {"rename":      {"field": "message2.result_count",         "target_field": "ldap_search.result_count",     "ignore_missing":true}},
+        {"rename":      {"field": "message2.filter",               "target_field": "ldap_search.filter",           "ignore_missing":true}},
+        {"rename":      {"field": "message2.attributes",           "target_field": "ldap_search.attributes",       "ignore_missing":true}},
+        {"script":      {"source": "if (ctx.containsKey('ldap') && ctx.ldap.containsKey('diagnostic_message') && ctx.ldap.diagnostic_message != null) {\n          String message = ctx.ldap.diagnostic_message;\n\n          // get user and property from SASL success\n          if (message.toLowerCase().contains(\"sasl(0): successful result\")) {\n            Pattern pattern = /user:\\s*([^ ]+)\\s*property:\\s*([^ ]+)/i;\n            Matcher matcher = pattern.matcher(message);\n            if (matcher.find()) {\n              ctx.ldap.user_email = matcher.group(1);  // Extract user email\n              ctx.ldap.property = matcher.group(2);    // Extract property\n            }\n          }\n          if (message.toLowerCase().contains(\"ldaperr:\")) {\n            Pattern pattern = /comment:\\s*([^,]+)/i;\n            Matcher matcher = pattern.matcher(message);\n\n            if (matcher.find()) {\n              ctx.ldap.comment = matcher.group(1);\n            }\n          }\n        }","ignore_failure": true}},
+        {"script":      {"source": "if (ctx.containsKey('ldap') && ctx.ldap.containsKey('object') && ctx.ldap.object != null) {\n    String message = ctx.ldap.object;\n\n    // parse common name from ldap object\n    if (message.toLowerCase().contains(\"cn=\")) {\n        Pattern pattern = /cn=([^,]+)/i;\n        Matcher matcher = pattern.matcher(message);\n        if (matcher.find()) {\n            ctx.ldap.common_name = matcher.group(1);  // Extract CN\n        }\n    }\n    // build domain from ldap object\n    if (message.toLowerCase().contains(\"dc=\")) {\n        Pattern dcPattern = /dc=([^,]+)/i;\n        Matcher dcMatcher = dcPattern.matcher(message);\n\n        StringBuilder domainBuilder = new StringBuilder();\n        while (dcMatcher.find()) {\n            if (domainBuilder.length() > 0 ){\n                domainBuilder.append(\".\");\n            }\n            domainBuilder.append(dcMatcher.group(1));\n        }\n        if (domainBuilder.length() > 0) {\n            ctx.ldap.domain = domainBuilder.toString();\n        }\n    }\n    // create list of any organizational units from ldap object\n    if (message.toLowerCase().contains(\"ou=\")) {\n        Pattern ouPattern = /ou=([^,]+)/i;\n        Matcher ouMatcher = ouPattern.matcher(message);\n        ctx.ldap.organizational_unit = [];\n\n        while (ouMatcher.find()) {\n            ctx.ldap.organizational_unit.add(ouMatcher.group(1));\n        }\n        if(ctx.ldap.organizational_unit.isEmpty()) {\n          ctx.remove(\"ldap.organizational_unit\");\n        }\n    }\n}\n","ignore_failure": true}},
+        {"remove":      {"field": "message2.tags",                 "ignore_failure": true}},
+        {"remove":      {"field": ["host"],                        "ignore_failure": true}},
+        {"pipeline":    {"name": "zeek.common"}}
+    ]
+}

salt/elasticsearch/files/ingest/zeek.ntp

@@ -0,0 +1,16 @@
+{
+  "description" : "zeek.ntp",
+  "processors":[
+    {"set":     {"field":"event.dataset",       "value":"ntp",                       "ignore_failure":true}},
+    {"json":    {"field":"message",             "target_field":"message2",           "ignore_failure":true}},
+    {"rename":  {"field":"message2.version",    "target_field":"ntp.version",        "ignore_missing":true}},
+    {"rename":  {"field":"message2.mode",       "target_field":"ntp.mode",           "ignore_missing":true}},
+    {"rename":  {"field":"message2.poll",       "target_field":"ntp.poll",           "ignore_missing":true}},
+    {"rename":  {"field":"message2.precision",  "target_field":"ntp.precision",      "ignore_missing":true}},
+    {"rename":  {"field":"message2.org_time",   "target_field":"ntp.org_time",       "ignore_missing":true}},
+    {"rename":  {"field":"message2.xmt_time",   "target_field":"ntp.xmt_time",       "ignore_missing":true}},
+    {"date":    {"field":"ntp.org_time",        "target_field":"ntp.org_time",  "formats":["UNIX", "UNIX_MS"], "ignore_failure": true, "if":"ctx?.ntp?.org_time != null"}},
+    {"date":    {"field":"ntp.xmt_time",        "target_field":"ntp.xmt_time",  "formats":["UNIX", "UNIX_MS"], "ignore_failure": true, "if":"ctx?.ntp?.xmt_time != null"}},
+    {"pipeline":{"name":"zeek.common"}}
+  ]
+}

salt/elasticsearch/files/ingest/zeek.quic

@@ -0,0 +1,18 @@
+{
+  "description" : "zeek.quic",
+  "processors" : [
+    { "set":      { "field": "event.dataset",                           "value": "quic" } },
+    { "set":      { "field": "network.transport",                       "value": "udp"  } },
+    { "json":     { "field": "message",                                 "target_field": "message2",                       "ignore_failure": true    } },
+    { "rename": 	{ "field": "message2.version",	                    "target_field": "quic.version",		              "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_initial_dcid",	        "target_field": "quic.client_initial_dcid",		  "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_scid",	                "target_field": "quic.client_scid",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_scid",	                "target_field": "quic.server_scid",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.server_name",	                "target_field": "quic.server_name",		          "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.client_protocol",	            "target_field": "quic.client_protocol",		      "ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.history",	                    "target_field": "quic.history",                   "ignore_missing": true 	} },
+    { "remove":		{ "field": "message2.tags",		                                                                      "ignore_failure": true    } },
+    { "remove":   { "field": ["host"],                                                                                    "ignore_failure": true    } },
+    { "pipeline":       { "name": "zeek.common" } }
+  ]
+}

salt/elasticsearch/files/ingest/zeek.software

@@ -11,7 +11,7 @@
     { "dot_expander": 	{ "field": "version.minor2", 		"path": "message2", 			"ignore_failure": true 	} },
     { "rename": 	{ "field": "message2.version.minor2", 	"target_field": "software.version.minor2",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "version.minor3", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.version.minor3", 	"target_field": "version.minor3",	"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.version.minor3", 	"target_field": "software.version.minor3",	"ignore_missing": true 	} },
     { "dot_expander": 	{ "field": "version.addl", 		"path": "message2", 			"ignore_failure": true 	} },
     { "rename": 	{ "field": "message2.version.addl", 	"target_field": "software.version.additional_info",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.host", 		"target_field": "source.ip",		"ignore_missing": true 	} },

salt/elasticsearch/files/ingest/zeek.traceroute

@@ -0,0 +1,10 @@
+{
+  "description":"zeek.traceroute",
+  "processors":[
+    {"set":         {"field":"event.dataset",       "value":"traceroute" }},
+    {"json":        {"field":"message",             "target_field":"message2" }},
+    {"rename":      {"field":"message2.src",        "target_field":"source.ip",         "ignore_missing":true,"ignore_failure":true}},
+    {"rename":      {"field":"message2.dst",        "target_field":"destination.ip",    "ignore_missing":true,"ignore_failure":true}},
+    {"pipeline":    {"name":"zeek.common"}}
+  ]
+}

salt/elasticsearch/soc_elasticsearch.yaml

@@ -1,7 +1,13 @@
 elasticsearch:
   enabled:
-    description: You can enable or disable Elasticsearch.
+    description: Enables or disables the Elasticsearch process. This process provides the log event storage system. WARNING - Disabling this process is unsupported.
+    advanced: True
     helpLink: elasticsearch.html
+  version:
+    description: "This specifies the version of the following containers: so-elastic-fleet-package-registry, so-elastic-agent, so-elastic-fleet, so-kibana, so-logstash and so-elasticsearch. Modifying this value in the Elasticsearch defaults.yaml will result in catastrophic grid failure."
+    readonly: True
+    global: True
+    advanced: True
   esheap:
     description: Specify the memory heap size in (m)egabytes for Elasticsearch.
     helpLink: elasticsearch.html
@@ -71,6 +77,13 @@ elasticsearch:
     custom008: *pipelines
     custom009: *pipelines
     custom010: *pipelines
+  managed_integrations:
+    description: List of integrations to add into SOC config UI. Enter the full or partial integration name. Eg. 1password, 1pass
+    forcedType: "[]string"
+    multiline: True
+    global: True
+    advanced: True
+    helpLink: elasticsearch.html
   index_settings:
     global_overrides:
       index_template:
@@ -120,7 +133,7 @@ elasticsearch:
                   helpLink: elasticsearch.html
           cold:
             min_age:
-              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier. While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
               regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
@@ -133,10 +146,11 @@ elasticsearch:
                   helpLink: elasticsearch.html
           warm:
             min_age: 
-              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier.
+              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
               regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
+              helpLink: elasticsearch.html
             actions:
               set_priority:
                 priority:
@@ -146,7 +160,7 @@ elasticsearch:
                   helpLink: elasticsearch.html
           delete:
             min_age:
-              description: Minimum age of index. ex. 90d - This determines when the index should be deleted.
+              description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
               regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
@@ -160,7 +174,7 @@ elasticsearch:
       index_template:
         index_patterns:
           description: Patterns for matching multiple indices or tables.
-          forceType: "[]string"
+          forcedType: "[]string"
           multiline: True
           global: True
           advanced: True
@@ -275,7 +289,7 @@ elasticsearch:
                   helpLink: elasticsearch.html
           warm:
             min_age:
-              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier.
+              description: Minimum age of index. ex. 30d - This determines when the index should be moved to the warm tier. Nodes in the warm tier generally don’t need to be as fast as those in the hot tier. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and warm min_age set to 30 then there will be 30 days from index creation to rollover and then an additional 30 days before moving to warm tier.
               regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
@@ -302,7 +316,7 @@ elasticsearch:
                   helpLink: elasticsearch.html
           cold:
             min_age:
-              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed.
+              description: Minimum age of index. ex. 60d - This determines when the index should be moved to the cold tier.  While still searchable, this tier is typically optimized for lower storage costs rather than search speed. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and cold min_age set to 60 then there will be 30 days from index creation to rollover and then an additional 60 days before moving to cold tier.
               regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
@@ -318,7 +332,7 @@ elasticsearch:
                   helpLink: elasticsearch.html
           delete:
             min_age:
-              description: Minimum age of index. This determines when the index should be deleted.
+              description: Minimum age of index. ex. 90d - This determines when the index should be deleted. It’s important to note that this is calculated relative to the rollover date (NOT the original creation date of the index). For example, if you have an index that is set to rollover after 30 days and delete min_age set to 90 then there will be 30 days from index creation to rollover and then an additional 90 days before deletion.
               regex: ^[0-9]{1,5}d$
               forcedType: string
               global: True
@@ -352,154 +366,13 @@ elasticsearch:
     so-logs-windows_x_powershell_operational: *indexSettings
     so-logs-windows_x_sysmon_operational: *indexSettings
     so-logs-winlog_x_winlog: *indexSettings
-    so-logs-apache_x_access: *indexSettings
-    so-logs-apache_x_error: *indexSettings
-    so-logs-auditd_x_log: *indexSettings
-    so-logs-aws_x_cloudtrail: *indexSettings
-    so-logs-aws_x_cloudwatch_logs: *indexSettings
-    so-logs-aws_x_ec2_logs: *indexSettings
-    so-logs-aws_x_elb_logs: *indexSettings
-    so-logs-aws_x_firewall_logs: *indexSettings
-    so-logs-aws_x_route53_public_logs: *indexSettings
-    so-logs-aws_x_route53_resolver_logs: *indexSettings
-    so-logs-aws_x_s3access: *indexSettings
-    so-logs-aws_x_vpcflow: *indexSettings
-    so-logs-aws_x_waf: *indexSettings
-    so-logs-azure_x_activitylogs: *indexSettings
-    so-logs-azure_x_application_gateway: *indexSettings
-    so-logs-azure_x_auditlogs: *indexSettings
-    so-logs-azure_x_eventhub: *indexSettings
-    so-logs-azure_x_firewall_logs: *indexSettings
-    so-logs-azure_x_identity_protection: *indexSettings
-    so-logs-azure_x_platformlogs: *indexSettings
-    so-logs-azure_x_provisioning: *indexSettings
-    so-logs-azure_x_signinlogs: *indexSettings
-    so-logs-azure_x_springcloudlogs: *indexSettings
-    so-logs-barracuda_x_waf: *indexSettings
-    so-logs-cef_x_log: *indexSettings
-    so-logs-cisco_asa_x_log: *indexSettings
-    so-logs-cisco_ftd_x_log: *indexSettings
-    so-logs-cisco_ios_x_log: *indexSettings
-    so-logs-cisco_ise_x_log: *indexSettings
-    so-logs-citrix_adc_x_interface: *indexSettings
-    so-logs-citrix_adc_x_lbvserver: *indexSettings
-    so-logs-citrix_adc_x_service: *indexSettings
-    so-logs-citrix_adc_x_system: *indexSettings
-    so-logs-citrix_adc_x_vpn: *indexSettings
-    so-logs-citrix_waf_x_log: *indexSettings
-    so-logs-cloudflare_x_audit: *indexSettings
-    so-logs-cloudflare_x_logpull: *indexSettings
-    so-logs-crowdstrike_x_falcon: *indexSettings
-    so-logs-crowdstrike_x_fdr: *indexSettings
-    so-logs-darktrace_x_ai_analyst_alert: *indexSettings
-    so-logs-darktrace_x_model_breach_alert: *indexSettings
-    so-logs-darktrace_x_system_status_alert: *indexSettings
     so-logs-detections_x_alerts: *indexSettings
-    so-logs-f5_bigip_x_log: *indexSettings
-    so-logs-fim_x_event: *indexSettings
-    so-logs-fortinet_x_clientendpoint: *indexSettings
-    so-logs-fortinet_x_firewall: *indexSettings
-    so-logs-fortinet_x_fortimail: *indexSettings
-    so-logs-fortinet_x_fortimanager: *indexSettings
-    so-logs-fortinet_x_fortigate: *indexSettings
-    so-logs-gcp_x_audit: *indexSettings
-    so-logs-gcp_x_dns: *indexSettings
-    so-logs-gcp_x_firewall: *indexSettings
-    so-logs-gcp_x_loadbalancing_logs: *indexSettings
-    so-logs-gcp_x_vpcflow: *indexSettings
-    so-logs-github_x_audit: *indexSettings
-    so-logs-github_x_code_scanning: *indexSettings
-    so-logs-github_x_dependabot: *indexSettings
-    so-logs-github_x_issues: *indexSettings
-    so-logs-github_x_secret_scanning: *indexSettings
-    so-logs-google_workspace_x_access_transparency: *indexSettings
-    so-logs-google_workspace_x_admin: *indexSettings
-    so-logs-google_workspace_x_alert: *indexSettings
-    so-logs-google_workspace_x_context_aware_access: *indexSettings
-    so-logs-google_workspace_x_device: *indexSettings
-    so-logs-google_workspace_x_drive: *indexSettings
-    so-logs-google_workspace_x_gcp: *indexSettings
-    so-logs-google_workspace_x_group_enterprise: *indexSettings
-    so-logs-google_workspace_x_groups: *indexSettings
-    so-logs-google_workspace_x_login: *indexSettings
-    so-logs-google_workspace_x_rules: *indexSettings
-    so-logs-google_workspace_x_saml: *indexSettings
-    so-logs-google_workspace_x_token: *indexSettings
-    so-logs-google_workspace_x_user_accounts: *indexSettings
     so-logs-http_endpoint_x_generic: *indexSettings
     so-logs-httpjson_x_generic: *indexSettings
-    so-logs-iis_x_access: *indexSettings
-    so-logs-iis_x_error: *indexSettings
-    so-logs-juniper_x_junos: *indexSettings
-    so-logs-juniper_x_netscreen: *indexSettings
-    so-logs-juniper_x_srx: *indexSettings
-    so-logs-juniper_srx_x_log: *indexSettings
-    so-logs-kafka_log_x_generic: *indexSettings
-    so-logs-lastpass_x_detailed_shared_folder: *indexSettings
-    so-logs-lastpass_x_event_report: *indexSettings
-    so-logs-lastpass_x_user: *indexSettings
-    so-logs-m365_defender_x_event: *indexSettings
-    so-logs-m365_defender_x_incident: *indexSettings
-    so-logs-m365_defender_x_log: *indexSettings
-    so-logs-microsoft_defender_endpoint_x_log: *indexSettings
-    so-logs-microsoft_dhcp_x_log: *indexSettings
-    so-logs-microsoft_sqlserver_x_audit: *indexSettings
-    so-logs-microsoft_sqlserver_x_log: *indexSettings
-    so-logs-mysql_x_error: *indexSettings
-    so-logs-mysql_x_slowlog: *indexSettings
-    so-logs-netflow_x_log: *indexSettings
-    so-logs-nginx_x_access: *indexSettings
-    so-logs-nginx_x_error: *indexSettings
-    so-logs-o365_x_audit: *indexSettings
-    so-logs-okta_x_system: *indexSettings
-    so-logs-panw_x_panos: *indexSettings
-    so-logs-pfsense_x_log: *indexSettings
-    so-logs-proofpoint_tap_x_clicks_blocked: *indexSettings
-    so-logs-proofpoint_tap_x_clicks_permitted: *indexSettings
-    so-logs-proofpoint_tap_x_message_blocked: *indexSettings
-    so-logs-proofpoint_tap_x_message_delivered: *indexSettings
-    so-logs-sentinel_one_x_activity: *indexSettings
-    so-logs-sentinel_one_x_agent: *indexSettings
-    so-logs-sentinel_one_x_alert: *indexSettings
-    so-logs-sentinel_one_x_group: *indexSettings
-    so-logs-sentinel_one_x_threat: *indexSettings
-    so-logs-sonicwall_firewall_x_log: *indexSettings
-    so-logs-snort_x_log: *indexSettings
-    so-logs-symantec_endpoint_x_log: *indexSettings
-    so-logs-tenable_io_x_asset: *indexSettings
-    so-logs-tenable_io_x_plugin: *indexSettings
-    so-logs-tenable_io_x_scan: *indexSettings
-    so-logs-tenable_io_x_vulnerability: *indexSettings
-    so-logs-tenable_sc_x_asset: *indexSettings
-    so-logs-tenable_sc_x_plugin: *indexSettings
-    so-logs-tenable_sc_x_vulnerability: *indexSettings
-    so-logs-ti_abusech_x_malware: *indexSettings
-    so-logs-ti_abusech_x_malwarebazaar: *indexSettings
-    so-logs-ti_abusech_x_threatfox: *indexSettings
-    so-logs-ti_abusech_x_url: *indexSettings
-    so-logs-ti_anomali_x_threatstream: *indexSettings
-    so-logs-ti_cybersixgill_x_threat: *indexSettings
-    so-logs-ti_misp_x_threat: *indexSettings
-    so-logs-ti_misp_x_threat_attributes: *indexSettings
-    so-logs-ti_otx_x_pulses_subscribed: *indexSettings  
-    so-logs-ti_otx_x_threat: *indexSettings
-    so-logs-ti_recordedfuture_x_latest_ioc-template: *indexSettings
-    so-logs-ti_recordedfuture_x_threat: *indexSettings
-    so-logs-ti_threatq_x_threat: *indexSettings
-    so-logs-zscaler_zia_x_alerts: *indexSettings
-    so-logs-zscaler_zia_x_dns: *indexSettings
-    so-logs-zscaler_zia_x_firewall: *indexSettings
-    so-logs-zscaler_zia_x_tunnel: *indexSettings
-    so-logs-zscaler_zia_x_web: *indexSettings
-    so-logs-zscaler_zpa_x_app_connector_status: *indexSettings
-    so-logs-zscaler_zpa_x_audit: *indexSettings
-    so-logs-zscaler_zpa_x_browser_access: *indexSettings
-    so-logs-zscaler_zpa_x_user_activity: *indexSettings
-    so-logs-zscaler_zpa_x_user_status: *indexSettings
-    so-logs-1password_x_item_usages: *indexSettings
-    so-logs-1password_x_signin_attempts: *indexSettings
     so-logs-osquery-manager-actions: *indexSettings
     so-logs-osquery-manager-action_x_responses: *indexSettings
+    so-logs-osquery-manager_x_action_x_responses: *indexSettings
+    so-logs-osquery-manager_x_result: *indexSettings
     so-logs-elastic_agent_x_apm_server: *indexSettings
     so-logs-elastic_agent_x_auditbeat: *indexSettings
     so-logs-elastic_agent_x_cloudbeat: *indexSettings
@@ -523,6 +396,9 @@ elasticsearch:
     so-metrics-endpoint_x_metrics: *indexSettings
     so-metrics-endpoint_x_policy: *indexSettings
     so-metrics-nginx_x_stubstatus: *indexSettings
+    so-metrics-vsphere_x_datastore: *indexSettings
+    so-metrics-vsphere_x_host: *indexSettings
+    so-metrics-vsphere_x_virtualmachine: *indexSettings
     so-case: *indexSettings
     so-common: *indexSettings
     so-endgame: *indexSettings
@@ -531,6 +407,7 @@ elasticsearch:
     so-suricata_x_alerts: *indexSettings
     so-import: *indexSettings
     so-kratos: *indexSettings
+    so-hydra: *indexSettings
     so-kismet: *indexSettings
     so-logstash: *indexSettings
     so-redis: *indexSettings

salt/elasticsearch/template.map.jinja

@@ -14,6 +14,18 @@
 
 {% set ES_INDEX_SETTINGS_ORIG = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings %}
 
+{# start generation of integration default index_settings #}
+{% if salt['file.file_exists']('/opt/so/state/esfleet_package_components.json') %}
+{%   set check_package_components = salt['file.stats']('/opt/so/state/esfleet_package_components.json') %}
+{%   if check_package_components.size > 1 %}
+{%    from 'elasticfleet/integration-defaults.map.jinja' import ADDON_INTEGRATION_DEFAULTS %}
+{%    for index, settings in ADDON_INTEGRATION_DEFAULTS.items() %}
+{%      do ES_INDEX_SETTINGS_ORIG.update({index: settings}) %}
+{%    endfor %}
+{%   endif%}
+{% endif %}
+{# end generation of integration default index_settings #}
+
 {% set ES_INDEX_SETTINGS_GLOBAL_OVERRIDES = {} %}
 {% for index in ES_INDEX_SETTINGS_ORIG.keys() %}
 {%   do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update({index: salt['defaults.merge'](ELASTICSEARCHDEFAULTS.elasticsearch.index_settings[index], PILLAR_GLOBAL_OVERRIDES, in_place=False)}) %}

salt/elasticsearch/templates/component/ecs/log.json

@@ -29,7 +29,7 @@
                 "file": {
                   "properties": {
                     "line": {
-                      "type": "integer"
+                      "type": "long"
                     },
                     "name": {
                       "ignore_above": 1024,

salt/elasticsearch/templates/component/ecs/metadata.json

@@ -0,0 +1,26 @@
+{
+  "template": {
+    "mappings": {
+      "dynamic_templates": [],
+      "properties": {
+        "metadata": {
+          "properties": {
+            "kafka": {
+              "properties": {
+                "timestamp": {
+                  "type": "date"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "_meta": {
+    "_meta": {
+      "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-log.html",
+      "ecs_version": "1.12.2"
+    }
+  }
+}

salt/elasticsearch/templates/component/ecs/zeek.json

@@ -603,6 +603,89 @@
                 }
               }
             },
+            "ipsec": {
+              "properties": {
+                "certificates": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "exchange_type": {
+                  "type": "short"
+                },
+                "flag_a": {
+                  "type": "boolean"
+                },
+                "flag_c": {
+                  "type": "boolean"
+                },
+                "flag_e": {
+                  "type": "boolean"
+                },
+                "flag_i": {
+                  "type": "boolean"
+                },
+                "flag_r": {
+                  "type": "boolean"
+                },
+                "flag_v": {
+                  "type": "boolean"
+                },
+                "hash": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "initiator_spi": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "ke_dh_groups": {
+                  "type": "short"
+                },
+                "length": {
+                  "type": "long"
+                },
+                "maj_version": {
+                  "type": "short"
+                },
+                "message_id": {
+                  "type": "long"
+                },
+                "min_version": {
+                  "type": "short"
+                },
+                "notify_messages": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "proposals": {
+                  "type": "long"
+                },
+                "responder_spi": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "situation": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "transform_attributes": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "transforms": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "vendor_ids": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "version": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            },
             "irc": {
               "properties": {
                 "addl": {
@@ -751,6 +834,81 @@
                 }
               }
             },
+            "ldap": {
+              "type": "object",
+              "properties": {
+                "message_id": {
+                  "type": "short"
+                },
+                "opcode": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "result": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "diagnostic_message": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "version": {
+                  "type": "short"
+                },
+                "object": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "argument": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "user_email": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "property": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "common_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "organizational_unit": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "domain": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            },
+            "ldap_search": {
+              "type": "object",
+              "properties": {
+                "scope": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "deref_aliases": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "result_count": {
+                  "type": "long"
+                },
+                "filter": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "attributes": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            },
             "modbus": {
               "properties": {
                 "exception": {
@@ -1089,6 +1247,38 @@
                 }
               }
             },
+            "quic": {
+              "type": "object",
+              "properties": {
+                "server_name": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "version": {
+                  "type": "short"
+                },
+                "client_initial_dcid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "client_scid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "server_scid": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "client_protocol": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "history": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                }
+              }
+            },
             "radius": {
               "properties": {
                 "connect_info": {

salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@custom.json

@@ -1,12 +0,0 @@
-{
-  "template": {
-    "settings": {}
-  },
-  "_meta": {
-    "package": {
-      "name": "elastic_agent"
-    },
-    "managed_by": "fleet",
-    "managed": true
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json

@@ -1,383 +0,0 @@
-{
-  "template": {
-    "settings": {
-      "index": {
-        "lifecycle": {
-          "name": "logs"
-        },
-        "codec": "best_compression",
-        "default_pipeline": "logs-elastic_agent-1.20.0",
-        "mapping": {
-          "total_fields": {
-            "limit": "10000"
-          }
-        },
-        "query": {
-          "default_field": [
-            "cloud.account.id",
-            "cloud.availability_zone",
-            "cloud.instance.id",
-            "cloud.instance.name",
-            "cloud.machine.type",
-            "cloud.provider",
-            "cloud.region",
-            "cloud.project.id",
-            "cloud.image.id",
-            "container.id",
-            "container.image.name",
-            "container.name",
-            "host.architecture",
-            "host.hostname",
-            "host.id",
-            "host.mac",
-            "host.name",
-            "host.os.family",
-            "host.os.kernel",
-            "host.os.name",
-            "host.os.platform",
-            "host.os.version",
-            "host.os.build",
-            "host.os.codename",
-            "host.type",
-            "ecs.version",
-            "agent.build.original",
-            "agent.ephemeral_id",
-            "agent.id",
-            "agent.name",
-            "agent.type",
-            "agent.version",
-            "log.level",
-            "message",
-            "elastic_agent.id",
-            "elastic_agent.process",
-            "elastic_agent.version",
-            "component.id",
-            "component.type",
-            "component.binary",
-            "component.state",
-            "component.old_state",
-            "unit.id",
-            "unit.type",
-            "unit.state",
-            "unit.old_state"
-          ]
-        }
-      }
-    },
-    "mappings": {
-      "dynamic": false,
-      "dynamic_templates": [
-        {
-          "container.labels": {
-            "path_match": "container.labels.*",
-            "mapping": {
-              "type": "keyword"
-            },
-            "match_mapping_type": "string"
-          }
-        }
-      ],
-      "properties": {
-        "container": {
-          "properties": {
-            "image": {
-              "properties": {
-                "name": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                }
-              }
-            },
-            "name": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "id": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        },
-        "agent": {
-          "properties": {
-            "build": {
-              "properties": {
-                "original": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                }
-              }
-            },
-            "name": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "id": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "ephemeral_id": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "type": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "version": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        },
-        "log": {
-          "properties": {
-            "level": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        },
-        "elastic_agent": {
-          "properties": {
-            "process": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "id": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "version": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "snapshot": {
-              "type": "boolean"
-            }
-          }
-        },
-        "message": {
-          "type": "text"
-        },
-        "cloud": {
-          "properties": {
-            "availability_zone": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "image": {
-              "properties": {
-                "id": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                }
-              }
-            },
-            "instance": {
-              "properties": {
-                "name": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "id": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                }
-              }
-            },
-            "provider": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "machine": {
-              "properties": {
-                "type": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                }
-              }
-            },
-            "project": {
-              "properties": {
-                "id": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                }
-              }
-            },
-            "region": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "account": {
-              "properties": {
-                "id": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                }
-              }
-            }
-          }
-        },
-        "component": {
-          "properties": {
-            "binary": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "old_state": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "id": {
-              "ignore_above": 1024,
-              "type": "wildcard"
-            },
-            "state": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "type": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        },
-        "unit": {
-          "properties": {
-            "old_state": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "id": {
-              "ignore_above": 1024,
-              "type": "wildcard"
-            },
-            "state": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "type": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        },
-        "@timestamp": {
-          "type": "date"
-        },
-        "ecs": {
-          "properties": {
-            "version": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        },
-        "data_stream": {
-          "properties": {
-            "namespace": {
-              "type": "constant_keyword"
-            },
-            "type": {
-              "type": "constant_keyword"
-            },
-            "dataset": {
-              "type": "constant_keyword"
-            }
-          }
-        },
-        "host": {
-          "properties": {
-            "hostname": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "os": {
-              "properties": {
-                "build": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "kernel": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "codename": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "name": {
-                  "ignore_above": 1024,
-                  "type": "keyword",
-                  "fields": {
-                    "text": {
-                      "type": "text"
-                    }
-                  }
-                },
-                "family": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "version": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                },
-                "platform": {
-                  "ignore_above": 1024,
-                  "type": "keyword"
-                }
-              }
-            },
-            "domain": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "ip": {
-              "type": "ip"
-            },
-            "containerized": {
-              "type": "boolean"
-            },
-            "name": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "id": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "type": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "mac": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "architecture": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        },
-        "event": {
-          "properties": {
-            "dataset": {
-              "type": "constant_keyword"
-            }
-          }
-        }
-      }
-    }
-  },
-  "_meta": {
-    "package": {
-      "name": "elastic_agent"
-    },
-    "managed_by": "fleet",
-    "managed": true
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@package.json

@@ -1,132 +0,0 @@
-{
-  "template": {
-    "settings": {
-      "index": {
-        "lifecycle": {
-          "name": "logs-endpoint.collection-diagnostic"
-        },
-        "codec": "best_compression",
-        "default_pipeline": "logs-endpoint.diagnostic.collection-8.10.2",
-        "mapping": {
-          "total_fields": {
-            "limit": "10000"
-          },
-          "ignore_malformed": "true"
-        },
-        "query": {
-          "default_field": [
-            "ecs.version",
-            "event.action",
-            "event.category",
-            "event.code",
-            "event.dataset",
-            "event.hash",
-            "event.id",
-            "event.kind",
-            "event.module",
-            "event.outcome",
-            "event.provider",
-            "event.type"
-          ]
-        }
-      }
-    },
-    "mappings": {
-      "dynamic": false,
-      "properties": {
-        "@timestamp": {
-          "ignore_malformed": false,
-          "type": "date"
-        },
-        "ecs": {
-          "properties": {
-            "version": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        },
-        "data_stream": {
-          "properties": {
-            "namespace": {
-              "type": "constant_keyword"
-            },
-            "type": {
-              "type": "constant_keyword"
-            },
-            "dataset": {
-              "type": "constant_keyword"
-            }
-          }
-        },
-        "event": {
-          "properties": {
-            "severity": {
-              "type": "long"
-            },
-            "code": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "created": {
-              "type": "date"
-            },
-            "kind": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "module": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "type": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "sequence": {
-              "type": "long"
-            },
-            "ingested": {
-              "type": "date"
-            },
-            "provider": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "action": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "id": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "category": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "dataset": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "hash": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "outcome": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        }
-      }
-    }
-  },
-  "_meta": {
-    "package": {
-      "name": "endpoint"
-    },
-    "managed_by": "fleet",
-    "managed": true
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/logs-osquery_manager.result@custom.json

@@ -0,0 +1,49 @@
+{
+  "template": {
+    "mappings": {
+      "dynamic_templates": [
+        {
+          "action_data.ecs_mapping": {
+            "path_match": "action_data.ecs_mapping.*",
+            "mapping": {
+              "type": "keyword"
+            },
+            "match_mapping_type": "string"
+          }
+        }
+      ],
+      "properties": {
+        "action_data": {
+          "dynamic": true,
+          "type": "object",
+          "properties": {
+            "ecs_mapping": {
+              "dynamic": true,
+              "type": "object"
+            },
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "platform": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "query": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "saved_query_id": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/elastic-agent/logs-soc@package.json

@@ -1,12 +1,10 @@
 {
-  "template": {
-    "settings": {}
-  },
+  "template": {},
   "_meta": {
     "package": {
-      "name": "endpoint"
+      "name": "log"
     },
     "managed_by": "fleet",
     "managed": true
   }
-}
+}

salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json

@@ -8,7 +8,35 @@
               "type": "match_only_text"
             }
 	  }
-	}
+	},
+	"host": {
+	  "properties":{
+            "ip": {
+              "type": "ip"
+            }
+	  }
+	},
+	"related": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+	"destination": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+	"source": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        }
       }
     }
   },

salt/elasticsearch/templates/component/elastic-agent/logs@custom.json

@@ -0,0 +1,9 @@
+{
+  "template": {
+    "settings": {
+      "index": {
+        "number_of_replicas": "0"
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_status@package.json

@@ -1,201 +0,0 @@
-{
-  "template": {
-    "settings": {
-      "index": {
-        "lifecycle": {
-          "name": "metrics"
-        },
-        "default_pipeline": "metrics-fleet_server.agent_status-1.5.0",
-        "mapping": {
-          "total_fields": {
-            "limit": "1000"
-          }
-        }
-      }
-    },
-    "mappings": {
-      "dynamic": false,
-      "_source": {
-        "mode": "synthetic"
-      },
-      "properties": {
-        "cluster": {
-          "properties": {
-            "id": {
-              "time_series_dimension": true,
-              "type": "keyword"
-            }
-          }
-        },
-        "fleet": {
-          "properties": {
-            "agents": {
-              "properties": {
-                "offline": {
-                  "time_series_metric": "gauge",
-                  "meta": {},
-                  "type": "long"
-                },
-                "total": {
-                  "time_series_metric": "gauge",
-                  "meta": {},
-                  "type": "long"
-                },
-                "updating": {
-                  "time_series_metric": "gauge",
-                  "meta": {},
-                  "type": "long"
-                },
-                "inactive": {
-                  "time_series_metric": "gauge",
-                  "meta": {},
-                  "type": "long"
-                },
-                "healthy": {
-                  "time_series_metric": "gauge",
-                  "meta": {},
-                  "type": "long"
-                },
-                "unhealthy": {
-                  "time_series_metric": "gauge",
-                  "meta": {},
-                  "type": "long"
-                },
-                "unenrolled": {
-                  "time_series_metric": "gauge",
-                  "meta": {},
-                  "type": "long"
-                },
-                "enrolled": {
-                  "time_series_metric": "gauge",
-                  "meta": {},
-                  "type": "long"
-                },
-                "unhealthy_reason": {
-                  "properties": {
-                    "output": {
-                      "time_series_metric": "gauge",
-                      "meta": {},
-                      "type": "long"
-                    },
-                    "input": {
-                      "time_series_metric": "gauge",
-                      "meta": {},
-                      "type": "long"
-                    },
-                    "other": {
-                      "time_series_metric": "gauge",
-                      "meta": {},
-                      "type": "long"
-                    }
-                  }
-                },
-                "upgrading_step": {
-                  "properties": {
-                    "rollback": {
-                      "time_series_metric": "gauge",
-                      "meta": {},
-                      "type": "long"
-                    },
-                    "requested": {
-                      "time_series_metric": "gauge",
-                      "meta": {},
-                      "type": "long"
-                    },
-                    "restarting": {
-                      "time_series_metric": "gauge",
-                      "meta": {},
-                      "type": "long"
-                    },
-                    "downloading": {
-                      "time_series_metric": "gauge",
-                      "meta": {},
-                      "type": "long"
-                    },
-                    "scheduled": {
-                      "time_series_metric": "gauge",
-                      "meta": {},
-                      "type": "long"
-                    },
-                    "extracting": {
-                      "time_series_metric": "gauge",
-                      "meta": {},
-                      "type": "long"
-                    },
-                    "replacing": {
-                      "time_series_metric": "gauge",
-                      "meta": {},
-                      "type": "long"
-                    },
-                    "failed": {
-                      "time_series_metric": "gauge",
-                      "meta": {},
-                      "type": "long"
-                    },
-                    "watching": {
-                      "time_series_metric": "gauge",
-                      "meta": {},
-                      "type": "long"
-                    }
-                  }
-                }
-              }
-            }
-          }
-        },
-        "agent": {
-          "properties": {
-            "id": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "type": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "version": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        },
-        "@timestamp": {
-          "ignore_malformed": false,
-          "type": "date"
-        },
-        "data_stream": {
-          "properties": {
-            "namespace": {
-              "type": "constant_keyword"
-            },
-            "type": {
-              "type": "constant_keyword"
-            },
-            "dataset": {
-              "type": "constant_keyword"
-            }
-          }
-        },
-        "kibana": {
-          "properties": {
-            "uuid": {
-              "path": "agent.id",
-              "type": "alias"
-            },
-            "version": {
-              "path": "agent.version",
-              "type": "alias"
-            }
-          }
-        }
-      }
-    }
-  },
-  "_meta": {
-    "package": {
-      "name": "fleet_server"
-    },
-    "managed_by": "fleet",
-    "managed": true
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/metrics-fleet_server.agent_versions@package.json

@@ -1,102 +0,0 @@
-{
-  "template": {
-    "settings": {
-      "index": {
-        "lifecycle": {
-          "name": "metrics"
-        },
-        "default_pipeline": "metrics-fleet_server.agent_versions-1.5.0",
-        "mapping": {
-          "total_fields": {
-            "limit": "1000"
-          }
-        }
-      }
-    },
-    "mappings": {
-      "dynamic": false,
-      "_source": {
-        "mode": "synthetic"
-      },
-      "properties": {
-        "cluster": {
-          "properties": {
-            "id": {
-              "time_series_dimension": true,
-              "type": "keyword"
-            }
-          }
-        },
-        "fleet": {
-          "properties": {
-            "agent": {
-              "properties": {
-                "count": {
-                  "time_series_metric": "gauge",
-                  "meta": {},
-                  "type": "long"
-                },
-                "version": {
-                  "time_series_dimension": true,
-                  "type": "keyword"
-                }
-              }
-            }
-          }
-        },
-        "agent": {
-          "properties": {
-            "id": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "type": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            },
-            "version": {
-              "ignore_above": 1024,
-              "type": "keyword"
-            }
-          }
-        },
-        "@timestamp": {
-          "ignore_malformed": false,
-          "type": "date"
-        },
-        "data_stream": {
-          "properties": {
-            "namespace": {
-              "type": "constant_keyword"
-            },
-            "type": {
-              "type": "constant_keyword"
-            },
-            "dataset": {
-              "type": "constant_keyword"
-            }
-          }
-        },
-        "kibana": {
-          "properties": {
-            "uuid": {
-              "path": "agent.id",
-              "type": "alias"
-            },
-            "version": {
-              "path": "agent.version",
-              "type": "alias"
-            }
-          }
-        }
-      }
-    }
-  },
-  "_meta": {
-    "package": {
-      "name": "fleet_server"
-    },
-    "managed_by": "fleet",
-    "managed": true
-  }
-}

salt/elasticsearch/templates/component/elastic-agent/metrics@custom.json

@@ -0,0 +1,9 @@
+{
+  "template": {
+    "settings": {
+      "index": {
+        "number_of_replicas": "0"
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/elastic-agent/so-fleet_globals-1.json

@@ -5,6 +5,7 @@
         "managed_by": "security_onion",
         "managed": true
       },
+      "date_detection": false,
       "dynamic_templates": [
         {
           "strings_as_keyword": {
@@ -16,7 +17,19 @@
           }
         }
       ],
-      "date_detection": false
+      "properties": {
+        "metadata": {
+          "properties": {
+            "kafka": {
+              "properties": {
+                "timestamp": {
+                  "type": "date"
+                }
+              }
+            }
+          }
+        }
+      }
     }
   },
   "_meta": {

salt/elasticsearch/templates/component/elastic-agent/so-fleet_integrations.ip_mappings-1.json

@@ -0,0 +1,59 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "host": {
+          "properties": {
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+        "related": {
+          "properties": {
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+        "destination": {
+          "properties": {
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+        "source": {
+          "properties": {
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
+        "metadata": {
+          "properties": {
+            "input": {
+              "properties": {
+                "beats": {
+                  "properties": {
+                    "host": {
+                      "properties": {
+                        "ip": {
+                          "type": "ip"
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "_meta": {
+    "managed_by": "security_onion",
+    "managed": true
+  }
+}

salt/elasticsearch/templates/component/so/detection-mappings.json

@@ -21,10 +21,10 @@
           "properties": {
             "publicId": {
               "ignore_above": 1024,
-	      "type": "keyword"
+              "type": "keyword"
             },
             "title": {
-	      "ignore_above": 1024,
+              "ignore_above": 1024,
               "type": "keyword"
             },
             "severity": {
@@ -38,15 +38,15 @@
             "description": {
               "type": "text"
             },
-	    "category": {
+            "category": {
               "ignore_above": 1024,
               "type": "keyword"
             },
-	    "product": {
+            "product": {
               "ignore_above": 1024,
               "type": "keyword"
             },
-	    "service": {
+            "service": {
               "ignore_above": 1024,
               "type": "keyword"
             },
@@ -64,7 +64,7 @@
             },
             "tags": {
               "ignore_above": 1024,
-	      "type": "keyword"
+              "type": "keyword"
             },
             "ruleset": {
               "ignore_above": 1024,
@@ -82,6 +82,12 @@
               "ignore_above": 1024,
               "type": "keyword"
             },
+            "sourceCreated": {
+              "type": "date"
+            },
+            "sourceUpdated": {
+              "type": "date"
+            },
             "overrides": {
               "properties": {
                 "type": {
@@ -97,6 +103,9 @@
                 "updatedAt": {
                   "type": "date"
                 },
+                "note": {
+                  "type": "text"
+                },
                 "regex": {
                   "type": "text"
                 },

salt/elasticsearch/templates/component/so/so-ip-mappings.json

@@ -0,0 +1,25 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-network.html",
+    "ecs_version": "1.12.2"
+  },
+  "template": {
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "so": {
+          "properties": {
+            "ip_address": {
+              "type": "ip"
+            },
+            "description": {
+              "type": "text"
+            }
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticsearch/templates/component/so/so-system-mappings.json

@@ -16,6 +16,13 @@
             }
           }
         },
+	"destination": {
+          "properties":{
+            "ip": {
+              "type": "ip"
+            }
+          }
+        },
 	"source": {
           "properties":{
             "ip": {

salt/elasticsearch/tools/sbin_jinja/so-catrust

@@ -6,13 +6,14 @@
 # Elastic License 2.0.
 
 . /usr/sbin/so-common
+get_elastic_agent_vars
 
 # Exit on errors, since all lines must succeed
 set -e
 
 # Check to see if we have extracted the ca cert.
 if [ ! -f /opt/so/saltstack/local/salt/elasticsearch/cacerts ]; then
-    docker run -v /etc/pki/ca.crt:/etc/ssl/ca.crt --name so-elasticsearchca --user root --entrypoint jdk/bin/keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ GLOBALS.so_version }} -keystore /usr/share/elasticsearch/jdk/lib/security/cacerts -alias SOSCA -import -file /etc/ssl/ca.crt -storepass changeit -noprompt
+    docker run -v /etc/pki/ca.crt:/etc/ssl/ca.crt --name so-elasticsearchca --user root --entrypoint jdk/bin/keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:$ELASTIC_AGENT_TARBALL_VERSION -keystore /usr/share/elasticsearch/jdk/lib/security/cacerts -alias SOSCA -import -file /etc/ssl/ca.crt -storepass changeit -noprompt
     docker cp so-elasticsearchca:/usr/share/elasticsearch/jdk/lib/security/cacerts /opt/so/saltstack/local/salt/elasticsearch/cacerts
     docker cp so-elasticsearchca:/etc/ssl/certs/ca-certificates.crt /opt/so/saltstack/local/salt/elasticsearch/tls-ca-bundle.pem
     docker rm so-elasticsearchca

salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load

@@ -23,6 +23,9 @@ else
   echo "This is the initial template load"
 fi
 
+# If soup is running, ignore errors
+pgrep soup > /dev/null && should_exit_on_failure=0
+
 load_failures=0
 
 load_template() {
@@ -30,7 +33,7 @@ load_template() {
   file=$2
 
   echo "Loading template file $i" 
-  if ! retry 3 5 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then
+  if ! retry 3 1 "so-elasticsearch-query $uri -d@$file -XPUT" "{\"acknowledged\":true}"; then
     if [[ $should_exit_on_failure -eq 1 ]]; then
       fail "Could not load template file: $file"
     else

salt/firewall/containers.map.jinja

@@ -9,6 +9,7 @@
      'so-influxdb',
      'so-kibana',
      'so-kratos',
+     'so-hydra',
      'so-nginx',
      'so-redis',
      'so-soc',
@@ -30,6 +31,7 @@
      'so-kafka',
      'so-kibana',
      'so-kratos',
+     'so-hydra',
      'so-logstash',
      'so-nginx',
      'so-redis',
@@ -73,6 +75,7 @@
      'so-influxdb',
      'so-kibana',
      'so-kratos',
+     'so-hydra',
      'so-nginx',
      'so-soc'
 ] %}

salt/firewall/defaults.yaml

@@ -10,6 +10,8 @@ firewall:
     elasticsearch_rest: []
     endgame: []
     eval: []
+    external_suricata: []
+    external_kafka: []
     fleet: []
     heavynode: []
     idh: []
@@ -86,6 +88,10 @@ firewall:
       tcp:
         - 3765
       udp: []
+    external_suricata:
+      tcp: 
+        - 7789
+      udp: []
     influxdb:
       tcp:
         - 8086
@@ -98,6 +104,10 @@ firewall:
       tcp:
         - 9092
       udp: []
+    kafka_external_access:
+      tcp:
+        - 29092
+      udp: []
     kibana:
       tcp:
         - 5601
@@ -216,6 +226,9 @@ firewall:
             analyst:
               portgroups:
                 - nginx
+            external_suricata:
+              portgroups:
+                - external_suricata
             customhostgroup0:
               portgroups: []
             customhostgroup1:
@@ -462,6 +475,11 @@ firewall:
             endgame:
               portgroups:
                 - endgame
+            external_suricata:
+              portgroups:
+                - external_suricata
+            external_kafka:
+              portgroups: []
             desktop:
               portgroups:
                 - docker_registry
@@ -654,6 +672,11 @@ firewall:
             endgame:
               portgroups:
                 - endgame
+            external_suricata:
+              portgroups:
+                - external_suricata
+            external_kafka:
+              portgroups: []
             desktop:
               portgroups:
                 - docker_registry
@@ -850,6 +873,11 @@ firewall:
             endgame:
               portgroups:
                 - endgame
+            external_suricata:
+              portgroups:
+                - external_suricata
+            external_kafka:
+              portgroups: []
             strelka_frontend:
               portgroups:
                 - strelka_frontend
@@ -1216,6 +1244,9 @@ firewall:
                 - elastic_agent_control
                 - elastic_agent_data
                 - elastic_agent_update
+            external_suricata:
+              portgroups:
+                - external_suricata
             analyst:
               portgroups:
                 - nginx
@@ -1317,6 +1348,8 @@ firewall:
             endgame:
               portgroups:
                 - endgame
+            external_kafka:
+              portgroups: []
             receiver:
               portgroups: []
             customhostgroup0:

salt/firewall/map.jinja

@@ -21,25 +21,38 @@
 {# Only add Kafka firewall items when Kafka enabled #}
 {% set role = GLOBALS.role.split('-')[1] %}
 
-{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone'] %}
-{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[role].portgroups.append('kafka_controller') %}
-{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
-{% endif %}
+{% if GLOBALS.pipeline == 'KAFKA' %}
+{%   set KAFKA_EXTERNAL_ACCESS = salt['pillar.get']('kafka:config:external_access:enabled', default=False) %}
+{%   set kafka_node_type = salt['pillar.get']('kafka:nodes:'+ GLOBALS.hostname + ':role') %}
 
-{% if GLOBALS.pipeline == 'KAFKA' and role == 'receiver' %}
-{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.self.portgroups.append('kafka_controller') %}
-{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.standalone.portgroups.append('kafka_controller') %}
-{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.manager.portgroups.append('kafka_controller') %}
-{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.managersearch.portgroups.append('kafka_controller') %}
-{%   do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
-{% endif %}
+{%   if role in ['manager', 'managersearch', 'standalone'] %}
+{%     do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[role].portgroups.append('kafka_controller') %}
+{%     do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
+{%   endif %}
 
-{% if GLOBALS.pipeline == 'KAFKA' and role in ['manager', 'managersearch', 'standalone', 'receiver'] %}
-{%   for r in ['manager', 'managersearch', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %}
-{%     if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %}
-{%       do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('kafka_data') %}
+{%   if role == 'receiver' %}
+{%     do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.self.portgroups.append('kafka_controller') %}
+{%     do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.standalone.portgroups.append('kafka_controller') %}
+{%     do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.manager.portgroups.append('kafka_controller') %}
+{%     do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.managersearch.portgroups.append('kafka_controller') %}
+{%     do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.receiver.portgroups.append('kafka_controller') %}
+{%   endif %}
+
+{%   if role in ['manager', 'managersearch', 'standalone', 'receiver'] %}
+{%     for r in ['manager', 'managersearch', 'standalone', 'receiver', 'fleet', 'idh', 'sensor', 'searchnode','heavynode', 'elastic_agent_endpoint', 'desktop'] %}
+{%       if FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r] is defined %}
+{%         do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups[r].portgroups.append('kafka_data') %}
+{%       endif %}
+{%     endfor %}
+{%   endif %}
+
+{%   if KAFKA_EXTERNAL_ACCESS %}
+{#     Kafka external access only applies for Kafka nodes with the broker role. #}
+{%     if role in ['manager', 'managersearch', 'standalone', 'receiver'] and 'broker' in kafka_node_type %}
+{%       do FIREWALL_DEFAULT.firewall.role[role].chain["DOCKER-USER"].hostgroups.external_kafka.portgroups.append('kafka_external_access') %}
 {%     endif %}
-{%   endfor %}
+{%   endif %}
+
 {% endif %}
 
-{% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %}
+{% set FIREWALL_MERGED = salt['pillar.get']('firewall', FIREWALL_DEFAULT.firewall, merge=True) %}

salt/firewall/soc_firewall.yaml

@@ -32,6 +32,8 @@ firewall:
     elasticsearch_rest: *hostgroupsettingsadv
     endgame: *hostgroupsettingsadv
     eval: *hostgroupsettings
+    external_suricata: *hostgroupsettings
+    external_kafka: *hostgroupsettings
     fleet: *hostgroupsettings
     heavynode: *hostgroupsettings
     idh: *hostgroupsettings
@@ -117,6 +119,9 @@ firewall:
     endgame:
       tcp: *tcpsettings
       udp: *udpsettings
+    external_suricata:
+      tcp: *tcpsettings
+      udp: *udpsettings
     influxdb:
       tcp: *tcpsettings
       udp: *udpsettings
@@ -126,6 +131,9 @@ firewall:
     kafka_data:
       tcp: *tcpsettings
       udp: *udpsettings
+    kafka_external_access:
+      tcp: *tcpsettings
+      udp: *udpsettings
     kibana:
       tcp: *tcpsettings
       udp: *udpsettings
@@ -215,6 +223,8 @@ firewall:
               portgroups: *portgroupsdocker
             elastic_agent_endpoint:
               portgroups: *portgroupsdocker
+            external_suricata: 
+              portgroups: *portgroupsdocker
             strelka_frontend:
               portgroups: *portgroupsdocker
             syslog:
@@ -370,6 +380,8 @@ firewall:
               portgroups: *portgroupsdocker
             endgame:
               portgroups: *portgroupsdocker
+            external_suricata: 
+              portgroups: *portgroupsdocker
             analyst:
               portgroups: *portgroupsdocker
             desktop:
@@ -463,6 +475,8 @@ firewall:
               portgroups: *portgroupsdocker
             analyst:
               portgroups: *portgroupsdocker
+            external_suricata: 
+              portgroups: *portgroupsdocker
             desktop:
               portgroups: *portgroupsdocker
             customhostgroup0:
@@ -554,6 +568,8 @@ firewall:
               portgroups: *portgroupsdocker
             endgame:
               portgroups: *portgroupsdocker
+            external_suricata: 
+              portgroups: *portgroupsdocker
             strelka_frontend:
               portgroups: *portgroupsdocker
             syslog:
@@ -828,6 +844,8 @@ firewall:
               portgroups: *portgroupsdocker
             analyst:
               portgroups: *portgroupsdocker
+            external_suricata: 
+              portgroups: *portgroupsdocker
             desktop:
               portgroups: *portgroupsdocker
             customhostgroup0: