watch some values

engines config for valueWatch
watch files change engine
2026-03-24 13:32:37 +01:00 · 2024-04-29 17:14:00 -04:00 · 2024-04-29 14:02:10 -04:00 · 2024-04-24 13:19:39 -04:00
752 changed files with 544699 additions and 34245 deletions
--- a/.github/.gitleaks.toml
+++ b/.github/.gitleaks.toml
@@ -536,7 +536,7 @@ secretGroup = 4

 [allowlist]
 description = "global allow lists"
-regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''', '''ssl_.*password''']
+regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''']
 paths = [
    '''gitleaks.toml''',
    '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',
--- a/.github/DISCUSSION_TEMPLATE/2-4.yml
+++ b/.github/DISCUSSION_TEMPLATE/2-4.yml
@@ -11,6 +11,7 @@ body:
      description: Which version of Security Onion 2.4.x are you asking about?
      options:
        -
+        - 2.4 Pre-release (Beta, Release Candidate)
        - 2.4.10
        - 2.4.20
        - 2.4.30
@@ -21,9 +22,6 @@ body:
        - 2.4.80
        - 2.4.90
        - 2.4.100
-        - 2.4.110
-        - 2.4.111
-        - 2.4.120
        - Other (please provide detail below)
    validations:
      required: true
@@ -34,10 +32,9 @@ body:
      options:
        -
        - Security Onion ISO image
-        - Cloud image (Amazon, Azure, Google)
-        - Network installation on Red Hat derivative like Oracle, Rocky, Alma, etc. (unsupported)
-        - Network installation on Ubuntu (unsupported)
-        - Network installation on Debian (unsupported)
+        - Network installation on Red Hat derivative like Oracle, Rocky, Alma, etc.
+        - Network installation on Ubuntu
+        - Network installation on Debian
        - Other (please provide detail below)
    validations:
      required: true
--- a/.github/workflows/close-threads.yml
+++ b/.github/workflows/close-threads.yml
@@ -15,7 +15,6 @@ concurrency:

 jobs:
  close-threads:
-    if: github.repository_owner == 'security-onion-solutions'
    runs-on: ubuntu-latest
    permissions:
      issues: write
--- a/.github/workflows/contrib.yml
+++ b/.github/workflows/contrib.yml
@@ -18,7 +18,7 @@ jobs:
        with:
          path-to-signatures: 'signatures_v1.json'
          path-to-document: 'https://securityonionsolutions.com/cla' 
-          allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,defensivedepth,m0duspwnens
+          allowlist: dependabot[bot],jertel,dougburks,TOoSmOotH,weslambert,defensivedepth,m0duspwnens
          remote-organization-name: Security-Onion-Solutions
          remote-repository-name: licensing

--- a/.github/workflows/lock-threads.yml
+++ b/.github/workflows/lock-threads.yml
@@ -15,7 +15,6 @@ concurrency:

 jobs:
  lock-threads:
-    if: github.repository_owner == 'security-onion-solutions'
    runs-on: ubuntu-latest
    steps:
      - uses: jertel/lock-threads@main
--- a/DOWNLOAD_AND_VERIFY_ISO.md
+++ b/DOWNLOAD_AND_VERIFY_ISO.md
@@ -1,17 +1,17 @@
-### 2.4.120-20250212 ISO image released on 2025/02/12
+### 2.4.60-20240320 ISO image released on 2024/03/20


 ### Download and Verify

-2.4.120-20250212 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.120-20250212.iso
+2.4.60-20240320 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.60-20240320.iso
 
-MD5: 3FF09F50AB1C9318CF0862DE9816102D  
-SHA1: 197AFA5A85C5CF95D0289FCD21BED7615FB8DB5C  
-SHA256: A59D94B09EEB39D8C2B6D0808792EC479B13D96FA7B32C3BEEFB6709C93F6692  
+MD5: 178DD42D06B2F32F3870E0C27219821E  
+SHA1: 73EDCD50817A7F6003FE405CF1808A30D034F89D  
+SHA256: DD334B8D7088A7B78160C253B680D645E25984BA5CCAB5CC5C327CA72137FC06  

 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.120-20250212.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.60-20240320.iso.sig

 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,29 +25,27 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.

 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.120-20250212.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.60-20240320.iso.sig
 ```

 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.120-20250212.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.60-20240320.iso
 ```

 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.120-20250212.iso.sig securityonion-2.4.120-20250212.iso
+gpg --verify securityonion-2.4.60-20240320.iso.sig securityonion-2.4.60-20240320.iso
 ```

 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Tue 11 Feb 2025 05:26:33 PM EST using RSA key ID FE507013
+gpg: Signature made Tue 19 Mar 2024 03:17:58 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.
 Primary key fingerprint: C804 A93D 36BE 0C73 3EA1  9644 7C10 60B7 FE50 7013
 ```

-If it fails to verify, try downloading again. If it still fails to verify, try downloading from another computer or another network.
-
 Once you've verified the ISO image, you're ready to proceed to our Installation guide:  
 https://docs.securityonion.net/en/2.4/installation.html
--- a/README.md
+++ b/README.md
@@ -8,22 +8,19 @@ Alerts
 ![Alerts](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/50_alerts.png)

 Dashboards
-![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_dashboards.png)
+![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/51_dashboards.png)

 Hunt
-![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/56_hunt.png)
-
-Detections
-![Detections](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_detections.png)
+![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/52_hunt.png)

 PCAP
-![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/62_pcap.png)
+![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_pcap.png)

 Grid
-![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/75_grid.png)
+![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_grid.png)

 Config
-![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/87_config.png)
+![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/61_config.png)

 ### Release Notes

--- a/SECURITY.md
+++ b/SECURITY.md
@@ -5,11 +5,9 @@
 | Version | Supported          |
 | ------- | ------------------ |
 | 2.4.x   | :white_check_mark: |
-| 2.3.x   | :x:                |
+| 2.3.x   | :white_check_mark: |
 | 16.04.x | :x:                |

-Security Onion 2.3 has reached End Of Life and is no longer supported.
-
 Security Onion 16.04 has reached End Of Life and is no longer supported.

 ## Reporting a Vulnerability
--- a/2
+++ b/2
@@ -1 +1 @@
-2.4.120
+2.4.70
--- a/pillar/elasticsearch/nodes.sls
+++ b/pillar/elasticsearch/nodes.sls
@@ -1,34 +0,0 @@
-{% set node_types = {} %}
-{% for minionid, ip in salt.saltutil.runner(
-    'mine.get',
-    tgt='elasticsearch:enabled:true',
-    fun='network.ip_addrs',
-    tgt_type='pillar') | dictsort()
-%}
-
-# only add a node to the pillar if it returned an ip from the mine
-{%   if ip | length > 0%}
-{%     set hostname = minionid.split('_') | first %}
-{%     set node_type = minionid.split('_') | last %}
-{%     if node_type not in node_types.keys() %}
-{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
-{%     else %}
-{%       if hostname not in node_types[node_type] %}
-{%         do node_types[node_type].update({hostname: ip[0]}) %}
-{%       else %}
-{%         do node_types[node_type][hostname].update(ip[0]) %}
-{%       endif %}
-{%     endif %}
-{%   endif %}
-{% endfor %}
-
-
-elasticsearch:
-  nodes:
-{% for node_type, values in node_types.items() %}
-    {{node_type}}:
-{%   for hostname, ip in values.items() %}
-      {{hostname}}:
-        ip: {{ip}}
-{%   endfor %}
-{% endfor %}
--- a/pillar/kafka/nodes.sls
+++ b/pillar/kafka/nodes.sls
@@ -1,2 +1,30 @@
+{% set current_kafkanodes = salt.saltutil.runner('mine.get', tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-receiver', fun='network.ip_addrs', tgt_type='compound') %}
+{% set pillar_kafkanodes = salt['pillar.get']('kafka:nodes', default={}, merge=True) %}
+
+{% set existing_ids = [] %}
+{% for node in pillar_kafkanodes.values() %}
+  {% if node.get('id') %}
+    {% do existing_ids.append(node['nodeid']) %}
+  {% endif %}
+{% endfor %}
+{% set all_possible_ids = range(1, 256)|list %}
+
+{% set available_ids = [] %}
+{% for id in all_possible_ids %}
+  {% if id not in existing_ids %}
+    {% do available_ids.append(id) %}
+  {% endif %}
+{% endfor %}
+
+{% set final_nodes = pillar_kafkanodes.copy() %}
+
+{% for minionid, ip in current_kafkanodes.items() %}
+    {% set hostname = minionid.split('_')[0] %}
+    {% if hostname not in final_nodes %}
+        {% set new_id = available_ids.pop(0) %}
+        {% do final_nodes.update({hostname: {'nodeid': new_id, 'ip': ip[0]}}) %}
+    {% endif %}
+{% endfor %}
+
 kafka:
-  nodes:
+  nodes: {{ final_nodes|tojson }}
--- a/pillar/logstash/nodes.sls
+++ b/pillar/logstash/nodes.sls
@@ -1,15 +1,16 @@
 {% set node_types = {} %}
+{% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %}
 {% for minionid, ip in salt.saltutil.runner(
    'mine.get',
-    tgt='logstash:enabled:true',
+    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ',
    fun='network.ip_addrs',
-    tgt_type='pillar') | dictsort()
+    tgt_type='compound') | dictsort()
 %}

 # only add a node to the pillar if it returned an ip from the mine
 {%   if ip | length > 0%}
-{%     set hostname = minionid.split('_') | first %}
-{%     set node_type = minionid.split('_') | last %}
+{%     set hostname = cached_grains[minionid]['host'] %}
+{%     set node_type = minionid.split('_')[1] %}
 {%     if node_type not in node_types.keys() %}
 {%       do node_types.update({node_type: {hostname: ip[0]}}) %}
 {%     else %}
--- a/pillar/redis/nodes.sls
+++ b/pillar/redis/nodes.sls
@@ -1,34 +0,0 @@
-{% set node_types = {} %}
-{% for minionid, ip in salt.saltutil.runner(
-    'mine.get',
-    tgt='redis:enabled:true',
-    fun='network.ip_addrs',
-    tgt_type='pillar') | dictsort()
-%}
-
-# only add a node to the pillar if it returned an ip from the mine
-{%   if ip | length > 0%}
-{%     set hostname = minionid.split('_') | first %}
-{%     set node_type = minionid.split('_') | last %}
-{%     if node_type not in node_types.keys() %}
-{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
-{%     else %}
-{%       if hostname not in node_types[node_type] %}
-{%         do node_types[node_type].update({hostname: ip[0]}) %}
-{%       else %}
-{%         do node_types[node_type][hostname].update(ip[0]) %}
-{%       endif %}
-{%     endif %}
-{%   endif %}
-{% endfor %}
-
-
-redis:
-  nodes:
-{% for node_type, values in node_types.items() %}
-    {{node_type}}:
-{%   for hostname, ip in values.items() %}
-      {{hostname}}:
-        ip: {{ip}}
-{%   endfor %}
-{% endfor %}
--- a/pillar/top.sls
+++ b/pillar/top.sls
@@ -16,8 +16,6 @@ base:
    - sensoroni.adv_sensoroni
    - telegraf.soc_telegraf
    - telegraf.adv_telegraf
-    - versionlock.soc_versionlock
-    - versionlock.adv_versionlock

  '* and not *_desktop':
    - firewall.soc_firewall
@@ -49,14 +47,10 @@ base:
    - kibana.adv_kibana
    - kratos.soc_kratos
    - kratos.adv_kratos
-    - hydra.soc_hydra
-    - hydra.adv_hydra
-    - redis.nodes
    - redis.soc_redis
    - redis.adv_redis
    - influxdb.soc_influxdb
    - influxdb.adv_influxdb
-    - elasticsearch.nodes
    - elasticsearch.soc_elasticsearch
    - elasticsearch.adv_elasticsearch
    - elasticfleet.soc_elasticfleet
@@ -100,7 +94,6 @@ base:
    - kibana.secrets
    {% endif %}
    - kratos.soc_kratos
-    - kratos.adv_kratos
    - elasticsearch.soc_elasticsearch
    - elasticsearch.adv_elasticsearch
    - elasticfleet.soc_elasticfleet
@@ -118,8 +111,8 @@ base:
    - kibana.adv_kibana
    - strelka.soc_strelka
    - strelka.adv_strelka
-    - hydra.soc_hydra
-    - hydra.adv_hydra
+    - kratos.soc_kratos
+    - kratos.adv_kratos
    - redis.soc_redis
    - redis.adv_redis
    - influxdb.soc_influxdb
@@ -154,14 +147,10 @@ base:
    - idstools.adv_idstools
    - kratos.soc_kratos
    - kratos.adv_kratos
-    - hydra.soc_hydra
-    - hydra.adv_hydra
-    - redis.nodes
    - redis.soc_redis
    - redis.adv_redis
    - influxdb.soc_influxdb
    - influxdb.adv_influxdb
-    - elasticsearch.nodes
    - elasticsearch.soc_elasticsearch
    - elasticsearch.adv_elasticsearch
    - elasticfleet.soc_elasticfleet
@@ -226,22 +215,17 @@ base:
    - logstash.nodes
    - logstash.soc_logstash
    - logstash.adv_logstash
-    - elasticsearch.nodes
    - elasticsearch.soc_elasticsearch
    - elasticsearch.adv_elasticsearch
    {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
    - elasticsearch.auth
    {% endif %}
-    - redis.nodes
    - redis.soc_redis
    - redis.adv_redis
    - minions.{{ grains.id }}
    - minions.adv_{{ grains.id }}
    - stig.soc_stig
    - soc.license
-    - kafka.nodes
-    - kafka.soc_kafka
-    - kafka.adv_kafka

  '*_receiver':
    - logstash.nodes
@@ -257,7 +241,6 @@ base:
    - kafka.nodes
    - kafka.soc_kafka
    - kafka.adv_kafka
-    - soc.license

  '*_import':
    - secrets
@@ -269,7 +252,6 @@ base:
    - kibana.secrets
    {% endif %}
    - kratos.soc_kratos
-    - kratos.adv_kratos
    - elasticsearch.soc_elasticsearch
    - elasticsearch.adv_elasticsearch
    - elasticfleet.soc_elasticfleet
@@ -285,8 +267,8 @@ base:
    - kibana.adv_kibana
    - backup.soc_backup
    - backup.adv_backup
-    - hydra.soc_hydra
-    - hydra.adv_hydra
+    - kratos.soc_kratos
+    - kratos.adv_kratos
    - redis.soc_redis
    - redis.adv_redis
    - influxdb.soc_influxdb
@@ -318,5 +300,3 @@ base:
  '*_desktop':
    - minions.{{ grains.id }}
    - minions.adv_{{ grains.id }}
-    - stig.soc_stig
-    - soc.license
--- a/pyci.sh
+++ b/pyci.sh
@@ -15,16 +15,12 @@ TARGET_DIR=${1:-.}

 PATH=$PATH:/usr/local/bin

-if [ ! -d .venv ]; then
-    python -m venv .venv
-fi
-
-source .venv/bin/activate
-
-if ! pip install flake8 pytest pytest-cov pyyaml; then
-    echo "Unable to install dependencies."
+if ! which pytest &> /dev/null || ! which flake8 &> /dev/null ; then
+    echo "Missing dependencies. Consider running the following command:"
+    echo "  python -m pip install flake8 pytest pytest-cov"
    exit 1
 fi

+pip install pytest pytest-cov
 flake8 "$TARGET_DIR" "--config=${HOME_DIR}/pytest.ini"
-python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100  "$TARGET_DIR" 
+python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100  "$TARGET_DIR" 
--- a/salt/allowed_states.map.jinja
+++ b/salt/allowed_states.map.jinja
@@ -24,7 +24,6 @@
          'influxdb',
          'soc',
          'kratos',
-          'hydra',
          'elasticfleet',
          'elastic-fleet-package-registry',
          'firewall',
@@ -66,10 +65,8 @@
          'registry',
          'manager',
          'nginx',
-          'strelka.manager',
          'soc',
          'kratos',
-          'hydra',
          'influxdb',
          'telegraf',
          'firewall',
@@ -94,10 +91,8 @@
          'nginx',
          'telegraf',
          'influxdb',
-          'strelka.manager',
          'soc',
          'kratos',
-          'hydra',
          'elasticfleet',
          'elastic-fleet-package-registry',
          'firewall',
@@ -117,10 +112,8 @@
          'nginx',
          'telegraf',
          'influxdb',
-          'strelka.manager',
          'soc',
          'kratos',
-          'hydra',
          'elastic-fleet-package-registry',
          'elasticfleet',
          'firewall',
@@ -140,9 +133,7 @@
          'firewall',
          'schedule',
          'docker_clean',
-          'stig',
-          'kafka.ca',
-          'kafka.ssl'
+          'stig'
          ],
      'so-standalone': [
          'salt.master',
@@ -155,7 +146,6 @@
          'influxdb',
          'soc',
          'kratos',
-          'hydra',
          'elastic-fleet-package-registry',
          'elasticfleet',
          'firewall',
@@ -202,13 +192,12 @@
          'schedule',
          'docker_clean',
          'kafka',
-          'stig'
+          'elasticsearch.ca'
          ],
      'so-desktop': [
          'ssl',
          'docker_clean',
-          'telegraf',
-          'stig'
+          'telegraf'
          ],
  }, grain='role') %}

--- a/salt/backup/defaults.yaml
+++ b/salt/backup/defaults.yaml
@@ -4,5 +4,4 @@ backup:
    - /etc/pki
    - /etc/salt
    - /nsm/kratos
-    - /nsm/hydra
  destination: "/nsm/backup"
--- a/salt/ca/files/signing_policies.conf
+++ b/salt/ca/files/signing_policies.conf
@@ -1,3 +1,6 @@
+mine_functions:
+  x509.get_pem_entries: [/etc/pki/ca.crt]
+
 x509_signing_policies:
  filebeat:
    - minions: '*'
--- a/salt/common/init.sls
+++ b/salt/common/init.sls
@@ -14,11 +14,6 @@ net.core.wmem_default:
  sysctl.present:
    - value: 26214400

-# Users are not a fan of console messages
-kernel.printk:
-  sysctl.present:
-    - value: "3 4 1 3"
-
 # Remove variables.txt from /tmp - This is temp
 rmvariablesfile:
  file.absent:
@@ -182,7 +177,6 @@ sostatus_log:
  file.managed:
    - name: /opt/so/log/sostatus/status.log
    - mode: 644
-    - replace: False

 # Install sostatus check cron. This is used to populate Grid.
 so-status_check_cron:
--- a/salt/common/soup_scripts.sls
+++ b/salt/common/soup_scripts.sls
@@ -1,8 +1,3 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
 {% if '2.4' in salt['cp.get_file_str']('/etc/soversion') %}

 {%   import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %}
@@ -11,7 +6,6 @@
 {%   else %}
 {%     set UPDATE_DIR='/tmp/sogh/securityonion' %}
 {%   endif %}
-{%   set SOVERSION = salt['file.read']('/etc/soversion').strip() %}

 remove_common_soup:
  file.absent:
@@ -21,8 +15,6 @@ remove_common_so-firewall:
  file.absent:
    - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall

-# This section is used to put the scripts in place in the Salt file system
-# in case a state run tries to overwrite what we do in the next section.
 copy_so-common_common_tools_sbin:
  file.copy:
    - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common
@@ -51,21 +43,6 @@ copy_so-firewall_manager_tools_sbin:
    - force: True
    - preserve: True

-copy_so-yaml_manager_tools_sbin:
-  file.copy:
-    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-yaml.py
-    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
-    - force: True
-    - preserve: True
-
-copy_so-repo-sync_manager_tools_sbin:
-  file.copy:
-    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-repo-sync
-    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
-    - preserve: True
-
-# This section is used to put the new script in place so that it can be called during soup.
-# It is faster than calling the states that normally manage them to put them in place.
 copy_so-common_sbin:
  file.copy:
    - name: /usr/sbin/so-common
@@ -101,24 +78,6 @@ copy_so-yaml_sbin:
    - force: True
    - preserve: True

-copy_so-repo-sync_sbin:
-  file.copy:
-    - name: /usr/sbin/so-repo-sync
-    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
-    - force: True
-    - preserve: True
-
-{# this is added in 2.4.120 to remove salt repo files pointing to saltproject.io to accomodate the move to broadcom and new bootstrap-salt script #}
-{%   if salt['pkg.version_cmp'](SOVERSION, '2.4.120') == -1 %}
-{%     set saltrepofile = '/etc/yum.repos.d/salt.repo' %}
-{%     if grains.os_family == 'Debian' %}
-{%       set saltrepofile = '/etc/apt/sources.list.d/salt.list' %}
-{%     endif %}
-remove_saltproject_io_repo_manager:
-  file.absent:
-    - name: {{ saltrepofile }}
-{%   endif %}
-
 {% else %}
 fix_23_soup_sbin:
  cmd.run:
--- a/salt/common/tools/sbin/so-checkin
+++ b/salt/common/tools/sbin/so-checkin
@@ -5,13 +5,8 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.

+
+
 . /usr/sbin/so-common

-cat << EOF
-
-so-checkin will run a full salt highstate to apply all salt states. If a highstate is already running, this request will be queued and so it may pause for a few minutes before you see any more output. For more information about so-checkin and salt, please see:
-https://docs.securityonion.net/en/2.4/salt.html
-
-EOF
-
-salt-call state.highstate -l info queue=True
+salt-call state.highstate -l info 
--- a/salt/common/tools/sbin/so-common
+++ b/salt/common/tools/sbin/so-common
@@ -8,6 +8,12 @@
 # Elastic agent is not managed by salt. Because of this we must store this base information in a 
 # script that accompanies the soup system. Since so-common is one of those special soup files, 
 # and since this same logic is required during installation, it's included in this file.
+ELASTIC_AGENT_TARBALL_VERSION="8.10.4"
+ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent

 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
@@ -25,11 +31,6 @@ if ! echo "$PATH" | grep -q "/usr/sbin"; then
  export PATH="$PATH:/usr/sbin"
 fi

-# See if a proxy is set. If so use it.
-if [ -f /etc/profile.d/so-proxy.sh ]; then
-  . /etc/profile.d/so-proxy.sh
-fi
-
 # Define a banner to separate sections
 banner="========================================================================="

@@ -168,46 +169,6 @@ check_salt_minion_status() {
 	return $status
 }

-# Compare es versions and return the highest version
-compare_es_versions() {
-    # Save the original IFS
-    local OLD_IFS="$IFS"
-
-    IFS=.
-    local i ver1=($1) ver2=($2)
-
-    # Restore the original IFS
-    IFS="$OLD_IFS"
-
-    # Determine the maximum length between the two version arrays
-    local max_len=${#ver1[@]}
-    if [[ ${#ver2[@]} -gt $max_len ]]; then
-        max_len=${#ver2[@]}
-    fi
-
-    # Compare each segment of the versions
-    for ((i=0; i<max_len; i++)); do
-		# If a segment in ver1 or ver2 is missing, set it to 0
-        if [[ -z ${ver1[i]} ]]; then
-            ver1[i]=0
-        fi
-        if [[ -z ${ver2[i]} ]]; then
-            ver2[i]=0
-        fi
-        if ((10#${ver1[i]} > 10#${ver2[i]})); then
-            echo "$1"
-            return 0
-        fi
-        if ((10#${ver1[i]} < 10#${ver2[i]})); then
-            echo "$2"
-            return 0
-        fi
-    done
-
-    echo "$1"  # If versions are equal, return either
-    return 0
-}
-
 copy_new_files() {
  # Copy new files over to the salt dir
  cd $UPDATE_DIR
@@ -218,21 +179,6 @@ copy_new_files() {
  cd /tmp
 }

-create_local_directories() {
-	echo "Creating local pillar and salt directories if needed"
-	PILLARSALTDIR=$1
-	local_salt_dir="/opt/so/saltstack/local"
-	for i in "pillar" "salt"; do
-		for d in $(find $PILLARSALTDIR/$i -type d); do
-			suffixdir=${d//$PILLARSALTDIR/}
-			if [ ! -d "$local_salt_dir/$suffixdir" ]; then
-				mkdir -pv $local_salt_dir$suffixdir
-			fi
-		done
-		chown -R socore:socore $local_salt_dir/$i
-	done
-}
-
 disable_fastestmirror() {
 	sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
 }
@@ -297,6 +243,11 @@ fail() {
 	exit 1
 }

+get_random_value() {
+	length=${1:-20}
+	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
+}
+
 get_agent_count() {
 	if [ -f /opt/so/log/agents/agentstatus.log ]; then
 		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}')
@@ -305,27 +256,6 @@ get_agent_count() {
 	fi
 }

-get_elastic_agent_vars() {
-	local path="${1:-/opt/so/saltstack/default}"
-	local defaultsfile="${path}/salt/elasticsearch/defaults.yaml"
-
-	if [ -f "$defaultsfile" ]; then
-		ELASTIC_AGENT_TARBALL_VERSION=$(egrep " +version: " $defaultsfile | awk -F: '{print $2}' | tr -d '[:space:]')
-		ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
-		ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
-		ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
-		ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
-		ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
-	else
-		fail "Could not find salt/elasticsearch/defaults.yaml"
-	fi
-}
-
-get_random_value() {
-	length=${1:-20}
-	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
-}
-
 gpg_rpm_import() {
 	if [[ $is_oracle ]]; then
 	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
@@ -677,8 +607,6 @@ has_uppercase() {
 }

 update_elastic_agent() {
-	local path="${1:-/opt/so/saltstack/default}"
-	get_elastic_agent_vars "$path"
 	echo "Checking if Elastic Agent update is necessary..."
 	download_and_verify "$ELASTIC_AGENT_URL" "$ELASTIC_AGENT_MD5_URL" "$ELASTIC_AGENT_FILE" "$ELASTIC_AGENT_MD5" "$ELASTIC_AGENT_EXPANSION_DIR"
 }
--- a/salt/common/tools/sbin/so-image-common
+++ b/salt/common/tools/sbin/so-image-common
@@ -29,7 +29,6 @@ container_list() {
      "so-influxdb"
      "so-kibana"
      "so-kratos"
-      "so-hydra"
      "so-nginx"
      "so-pcaptools"
      "so-soc"
@@ -54,7 +53,6 @@ container_list() {
      "so-kafka"
      "so-kibana"
      "so-kratos"
-      "so-hydra"
      "so-logstash"
      "so-nginx"
      "so-pcaptools"
@@ -114,10 +112,6 @@ update_docker_containers() {
    container_list
  fi

-  # all the images using ELASTICSEARCHDEFAULTS.elasticsearch.version
-  # does not include so-elastic-fleet since that container uses so-elastic-agent image
-  local IMAGES_USING_ES_VERSION=("so-elasticsearch")
-
  rm -rf $SIGNPATH >> "$LOG_FILE" 2>&1 
  mkdir -p $SIGNPATH >> "$LOG_FILE" 2>&1 

@@ -145,36 +139,15 @@ update_docker_containers() {
      $PROGRESS_CALLBACK $i
    fi

-    if [[ " ${IMAGES_USING_ES_VERSION[*]} " =~ [[:space:]]${i}[[:space:]] ]]; then
-      # this is an es container so use version defined in elasticsearch defaults.yaml
-      local UPDATE_DIR='/tmp/sogh/securityonion'
-      if [ ! -d "$UPDATE_DIR" ]; then
-        UPDATE_DIR=/securityonion
-      fi
-      local v1=0
-      local v2=0
-      if [[ -f "$UPDATE_DIR/salt/elasticsearch/defaults.yaml" ]]; then
-        v1=$(egrep " +version: " "$UPDATE_DIR/salt/elasticsearch/defaults.yaml" | awk -F: '{print $2}' | tr -d '[:space:]')
-      fi
-      if [[ -f "$DEFAULT_SALT_DIR/salt/elasticsearch/defaults.yaml" ]]; then
-        v2=$(egrep " +version: " "$DEFAULT_SALT_DIR/salt/elasticsearch/defaults.yaml" | awk -F: '{print $2}' | tr -d '[:space:]')
-      fi
-      local highest_es_version=$(compare_es_versions "$v1" "$v2")
-      local image=$i:$highest_es_version$IMAGE_TAG_SUFFIX
-      local sig_url=https://sigs.securityonion.net/es-$highest_es_version/$image.sig
-    else
-      # this is not an es container so use the so version for the version
-      local image=$i:$VERSION$IMAGE_TAG_SUFFIX
-      local sig_url=https://sigs.securityonion.net/$VERSION/$image.sig
-    fi
    # Pull down the trusted docker image
+    local image=$i:$VERSION$IMAGE_TAG_SUFFIX
    run_check_net_err \
    "docker pull $CONTAINER_REGISTRY/$IMAGEREPO/$image" \
    "Could not pull $image, please ensure connectivity to $CONTAINER_REGISTRY" >> "$LOG_FILE" 2>&1 
    
    # Get signature
    run_check_net_err \
-    "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' $sig_url --output $SIGNPATH/$image.sig" \
+    "curl --retry 5 --retry-delay 60 -A '$CURLTYPE/$CURRENTVERSION/$OS/$(uname -r)' https://sigs.securityonion.net/$VERSION/$i:$VERSION$IMAGE_TAG_SUFFIX.sig --output $SIGNPATH/$image.sig" \
    "Could not pull signature file for $image, please ensure connectivity to https://sigs.securityonion.net " \
    noretry >> "$LOG_FILE" 2>&1
    # Dump our hash values
--- a/salt/common/tools/sbin/so-log-check
+++ b/salt/common/tools/sbin/so-log-check
@@ -95,8 +95,6 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|shutdown process"             # server not yet ready (logstash waiting on elastic)
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|contain valid certificates"   # server not yet ready (logstash waiting on elastic)
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failedaction"                 # server not yet ready (logstash waiting on elastic)
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|block in start_workers"       # server not yet ready (logstash waiting on elastic)
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|block in buffer_initialize"   # server not yet ready (logstash waiting on elastic)
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no route to host"             # server not yet ready
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not running"                  # server not yet ready
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unavailable"                  # server not yet ready
@@ -125,7 +123,6 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tls handshake error"          # Docker registry container when new node comes onlines
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to get license information" # Logstash trying to contact ES before it's ready
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process already finished"     # Telegraf script finished just as the auto kill timeout kicked in
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|No shard available"           # Typical error when making a query before ES has finished loading all indices
 fi

 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -150,8 +147,6 @@ if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200"                   # false positive (request successful, contained error string in content)
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error"              # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal"  # false positive (Open Canary logging out blank IP addresses)
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncing rule"                 # false positive (rule sync log line includes rule name which can contain 'error') 
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request_unauthorized"         # false positive (login failures to Hydra result in an 'error' log) 
 fi

 if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
@@ -175,7 +170,6 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cannot join on an empty table" # InfluxDB flux query, import nodes
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|exhausting result iterator"   # InfluxDB flux query mismatched table results (temporary data issue)
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to finish run"         # InfluxDB rare error, self-recoverable
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to gather disk name"   # InfluxDB known error, can't read disks because the container doesn't have them mounted
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration"
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets"
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed"
@@ -207,12 +201,6 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unknown column"               # Elastalert errors from running EQL queries
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parsing_exception"            # Elastalert EQL parsing issue. Temp.
    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded"
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error running query:"         # Specific issues with detection rules
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|detect-parse"                 # Suricata encountering a malformed rule
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed"       # Detections: Exclude false positive due to automated testing
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors"                   # Detections: Not an actual error
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Initialized license manager"  # SOC log: before fields.status was changed to fields.licenseStatus
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|from NIC checksum offloading" # zeek reporter.log
 fi

 RESULT=0
@@ -248,11 +236,6 @@ exclude_log "playbook.log" # Playbook is removed as of 2.4.70, logs may still be
 exclude_log "mysqld.log"   # MySQL is removed as of 2.4.70, logs may still be on disk
 exclude_log "soctopus.log" # Soctopus is removed as of 2.4.70, logs may still be on disk
 exclude_log "agentstatus.log" # ignore this log since it tracks agents in error state
-exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable
-exclude_log "/nsm/kafka/data/" # ignore Kafka data directory from log check.
-
-# Include Zeek reporter.log to detect errors after running known good pcap(s) through sensor
-echo "/nsm/zeek/spool/logger/reporter.log" >> /tmp/log_check_files

 for log_file in $(cat /tmp/log_check_files); do
    status "Checking log file $log_file"
--- a/salt/common/tools/sbin/so-luks-tpm-regen
+++ b/salt/common/tools/sbin/so-luks-tpm-regen
@@ -1,98 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0."
-
-set -e
-# This script is intended to be used in the case the ISO install did not properly setup TPM decrypt for LUKS partitions at boot.
-if [ -z $NOROOT ]; then
-	# Check for prerequisites
-	if [ "$(id -u)" -ne 0 ]; then
-		echo "This script must be run using sudo!"
-		exit 1
-	fi
-fi
-ENROLL_TPM=N
-
-while [[ $# -gt 0 ]]; do
-  case $1 in
-    --enroll-tpm)
-      ENROLL_TPM=Y
-      ;;
-    *)
-      echo "Usage: $0 [options]"
-      echo ""
-      echo "where options are:"
-      echo "  --enroll-tpm            for when TPM enrollment was not selected during ISO install."
-      echo ""
-      exit 1
-      ;;
-  esac
-  shift
-done
-
-check_for_tpm() {
-  echo -n "Checking for TPM: "
-  if [ -d /sys/class/tpm/tpm0 ]; then
-    echo -e "tpm0 found."
-    TPM="yes"
-    # Check if TPM is using sha1 or sha256
-    if [ -d /sys/class/tpm/tpm0/pcr-sha1 ]; then
-      echo -e "TPM is using sha1.\n"
-      TPM_PCR="sha1"
-    elif [ -d /sys/class/tpm/tpm0/pcr-sha256 ]; then
-      echo -e "TPM is using sha256.\n"
-      TPM_PCR="sha256"
-    fi
-  else
-    echo -e "No TPM found.\n"
-    exit 1
-  fi
-}
-
-check_for_luks_partitions() {
-  echo "Checking for LUKS partitions"
-  for part in $(lsblk -o NAME,FSTYPE -ln | grep crypto_LUKS | awk '{print $1}'); do
-    echo "Found LUKS partition: $part"
-    LUKS_PARTITIONS+=("$part")
-  done
-  if [ ${#LUKS_PARTITIONS[@]} -eq 0 ]; then
-    echo -e "No LUKS partitions found.\n"
-    exit 1
-  fi
-  echo ""
-}
-
-enroll_tpm_in_luks() {
-  read -s -p "Enter the LUKS passphrase used during ISO install: " LUKS_PASSPHRASE
-  echo ""
-  for part in "${LUKS_PARTITIONS[@]}"; do
-    echo "Enrolling TPM for LUKS device: /dev/$part"
-    if [ "$TPM_PCR" == "sha1" ]; then
-      clevis luks bind -d /dev/$part tpm2 '{"pcr_bank":"sha1","pcr_ids":"7"}' <<< $LUKS_PASSPHRASE
-    elif [ "$TPM_PCR" == "sha256" ]; then
-      clevis luks bind -d /dev/$part tpm2 '{"pcr_bank":"sha256","pcr_ids":"7"}' <<< $LUKS_PASSPHRASE
-    fi
-  done
- }
-
-regenerate_tpm_enrollment_token() {
-  for part in "${LUKS_PARTITIONS[@]}"; do
-    clevis luks regen -d /dev/$part -s 1 -q
-  done
-}
-
-check_for_tpm
-check_for_luks_partitions
-
-if [[ $ENROLL_TPM == "Y" ]]; then
-  enroll_tpm_in_luks
-else
-  regenerate_tpm_enrollment_token
-fi
-
-echo "Running dracut"
-dracut -fv
-echo -e "\nTPM configuration complete. Reboot the system to verify the TPM is correctly decrypting the LUKS partition(s) at boot.\n"
--- a/salt/common/tools/sbin_jinja/so-tcpreplay
+++ b/salt/common/tools/sbin_jinja/so-tcpreplay
@@ -10,7 +10,7 @@
 . /usr/sbin/so-common
 . /usr/sbin/so-image-common

-REPLAYIFACE=${REPLAYIFACE:-"{{salt['pillar.get']('sensor:interface', '')}}"}
+REPLAYIFACE=${REPLAYIFACE:-$(lookup_pillar interface sensor)}
 REPLAYSPEED=${REPLAYSPEED:-10}

 mkdir -p /opt/so/samples
--- a/salt/common/tools/sbin_jinja/so-import-pcap
+++ b/salt/common/tools/sbin_jinja/so-import-pcap
@@ -89,7 +89,6 @@ function suricata() {
    -v ${LOG_PATH}:/var/log/suricata/:rw \
    -v ${NSM_PATH}/:/nsm/:rw \
    -v "$PCAP:/input.pcap:ro" \
-    -v /dev/null:/nsm/suripcap:rw \
    -v /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro \
    {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }} \
    --runmode single -k none -r /input.pcap > $LOG_PATH/console.log 2>&1
@@ -248,7 +247,7 @@ fi
 START_OLDEST_SLASH=$(echo $START_OLDEST | sed -e 's/-/%2F/g')
 END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
 if [[ $VALID_PCAPS_COUNT -gt 0 ]] || [[ $SKIPPED_PCAPS_COUNT -gt 0 ]]; then
-  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20event.module*%20%7C%20groupby%20-sankey%20event.module*%20event.dataset%20%7C%20groupby%20event.dataset%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20network.protocol%20%7C%20groupby%20rule.name%20rule.category%20event.severity_label%20%7C%20groupby%20dns.query.name%20%7C%20groupby%20file.mime_type%20%7C%20groupby%20http.virtual_host%20http.uri%20%7C%20groupby%20notice.note%20notice.message%20notice.sub_message%20%7C%20groupby%20ssl.server_name%20%7C%20groupby%20source_geo.organization_name%20source.geo.country_name%20%7C%20groupby%20destination_geo.organization_name%20destination.geo.country_name&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
+  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20-sankey%20event.dataset%20event.category%2a%20%7C%20groupby%20-pie%20event.category%20%7C%20groupby%20-bar%20event.module%20%7C%20groupby%20event.dataset%20%7C%20groupby%20event.module%20%7C%20groupby%20event.category%20%7C%20groupby%20observer.name%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"

  status "Import complete!"
  status
--- a/salt/common/tools/sbin_jinja/so-raid-status
+++ b/salt/common/tools/sbin_jinja/so-raid-status
@@ -9,9 +9,6 @@

 . /usr/sbin/so-common

-software_raid=("SOSMN" "SOSMN-DE02" "SOSSNNV" "SOSSNNV-DE02" "SOS10k-DE02" "SOS10KNV" "SOS10KNV-DE02" "SOS10KNV-DE02" "SOS2000-DE02" "SOS-GOFAST-LT-DE02" "SOS-GOFAST-MD-DE02" "SOS-GOFAST-HV-DE02")
-hardware_raid=("SOS1000" "SOS1000F" "SOSSN7200" "SOS5000" "SOS4000")
-
 {%- if salt['grains.get']('sosmodel', '') %}
 {%- set model = salt['grains.get']('sosmodel') %}
 model={{ model }}
@@ -19,42 +16,33 @@ model={{ model }}
 if [[ $model =~ ^(SO2AMI01|SO2AZI01|SO2GCI01)$ ]]; then
    exit 0
 fi
-
-for i in "${software_raid[@]}"; do
-  if [[ "$model" == $i ]]; then
-    is_softwareraid=true
-    is_hwraid=false
-    break
-  fi
-done
-
-for i in "${hardware_raid[@]}"; do
-  if [[ "$model" == $i ]]; then
-    is_softwareraid=false
-    is_hwraid=true
-    break
-  fi
-done
-
 {%- else %}
 echo "This is not an appliance"
 exit 0
 {%- endif %}
+if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200|SOSSNNV|SOSMN)$ ]]; then
+	is_bossraid=true
+fi
+if [[ $model =~ ^(SOSSNNV|SOSMN)$ ]]; then
+	is_swraid=true
+fi
+if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200)$ ]]; then
+	is_hwraid=true
+fi

 check_nsm_raid() {
  PERCCLI=$(/opt/raidtools/perccli/perccli64 /c0/v0 show|grep RAID|grep Optl)
  MEGACTL=$(/opt/raidtools/megasasctl |grep optimal)
-  if [[ "$model" == "SOS500" || "$model" == "SOS500-DE02" ]]; then
-    #This doesn't have raid
-    HWRAID=0
-  else
+
+  if [[ $APPLIANCE == '1' ]]; then
    if [[ -n $PERCCLI ]]; then
      HWRAID=0
    elif [[ -n $MEGACTL ]]; then
      HWRAID=0
    else
      HWRAID=1
-    fi   
+    fi
+
  fi

 }
@@ -62,27 +50,17 @@ check_nsm_raid() {
 check_boss_raid() {
  MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional)
  MVTEST=$(/usr/local/bin/mvcli info -o vd | grep "No adapter")
-  BOSSNVMECLI=$(/usr/local/bin/mnv_cli info -o vd -i 0 | grep Functional)

-  # Is this NVMe Boss Raid?
-  if [[ "$model" =~ "-DE02" ]]; then
-    if [[ -n $BOSSNVMECLI ]]; then
+  # Check to see if this is a SM based system
+  if [[ -z $MVTEST ]]; then
+    if [[ -n $MVCLI ]]; then
      BOSSRAID=0
    else
      BOSSRAID=1
    fi
  else
-    # Check to see if this is a SM based system
-    if [[ -z $MVTEST ]]; then
-      if [[ -n $MVCLI ]]; then
-        BOSSRAID=0
-      else
-        BOSSRAID=1
-      fi
-    else
-      # This doesn't have boss raid so lets make it 0
-      BOSSRAID=0
-    fi
+    # This doesn't have boss raid so lets make it 0
+    BOSSRAID=0
  fi
 }

@@ -101,13 +79,14 @@ SWRAID=0
 BOSSRAID=0
 HWRAID=0

-if [[ "$is_hwraid" == "true" ]]; then
+if [[ $is_hwraid ]]; then
 	check_nsm_raid
-  check_boss_raid
 fi
-if [[ "$is_softwareraid" == "true" ]]; then
+if [[ $is_bossraid ]]; then
+	check_boss_raid
+fi
+if [[ $is_swraid ]]; then
 	check_software_raid
-  check_boss_raid
 fi

 sum=$(($SWRAID + $BOSSRAID + $HWRAID))
--- a/salt/docker/defaults.yaml
+++ b/salt/docker/defaults.yaml
@@ -51,14 +51,6 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-    'so-hydra':
-      final_octet: 30
-      port_bindings:
-        - 0.0.0.0:4444:4444
-        - 0.0.0.0:4445:4445
-      custom_bind_mounts: []
-      extra_hosts: []
-      extra_env: []
    'so-logstash':
      final_octet: 29
      port_bindings:
@@ -82,7 +74,6 @@ docker:
        - 443:443
        - 8443:8443
        - 7788:7788
-        - 7789:7789
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
@@ -189,8 +180,6 @@ docker:
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
-      ulimits:
-        - memlock=524288000
    'so-zeek':
      final_octet: 99
      custom_bind_mounts: []
@@ -201,7 +190,6 @@ docker:
      port_bindings:
        - 0.0.0.0:9092:9092
        - 0.0.0.0:9093:9093
-        - 0.0.0.0:8778:8778
      custom_bind_mounts: []
      extra_hosts: []
      extra_env: []
--- a/salt/docker/init.sls
+++ b/salt/docker/init.sls
@@ -20,41 +20,41 @@ dockergroup:
 dockerheldpackages:
  pkg.installed:
    - pkgs:
-      - containerd.io: 1.7.21-1
-      - docker-ce: 5:27.2.0-1~debian.12~bookworm
-      - docker-ce-cli: 5:27.2.0-1~debian.12~bookworm
-      - docker-ce-rootless-extras: 5:27.2.0-1~debian.12~bookworm
+      - containerd.io: 1.6.21-1
+      - docker-ce: 5:24.0.3-1~debian.12~bookworm
+      - docker-ce-cli: 5:24.0.3-1~debian.12~bookworm
+      - docker-ce-rootless-extras: 5:24.0.3-1~debian.12~bookworm
    - hold: True
    - update_holds: True
 {%    elif grains.oscodename == 'jammy' %}
 dockerheldpackages:
  pkg.installed:
    - pkgs:
-      - containerd.io: 1.7.21-1
-      - docker-ce: 5:27.2.0-1~ubuntu.22.04~jammy
-      - docker-ce-cli: 5:27.2.0-1~ubuntu.22.04~jammy
-      - docker-ce-rootless-extras: 5:27.2.0-1~ubuntu.22.04~jammy
+      - containerd.io: 1.6.21-1
+      - docker-ce: 5:24.0.2-1~ubuntu.22.04~jammy
+      - docker-ce-cli: 5:24.0.2-1~ubuntu.22.04~jammy
+      - docker-ce-rootless-extras: 5:24.0.2-1~ubuntu.22.04~jammy
    - hold: True
    - update_holds: True
 {%    else %}
 dockerheldpackages:
  pkg.installed:
    - pkgs:
-      - containerd.io: 1.7.21-1
-      - docker-ce: 5:27.2.0-1~ubuntu.20.04~focal
-      - docker-ce-cli: 5:27.2.0-1~ubuntu.20.04~focal
-      - docker-ce-rootless-extras: 5:27.2.0-1~ubuntu.20.04~focal
+      - containerd.io: 1.4.9-1
+      - docker-ce: 5:20.10.8~3-0~ubuntu-focal
+      - docker-ce-cli: 5:20.10.5~3-0~ubuntu-focal
+      - docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-focal
    - hold: True
    - update_holds: True
-{%   endif %}
+{% endif %}
 {% else %}
 dockerheldpackages:
  pkg.installed:
    - pkgs:
-      - containerd.io: 1.7.21-3.1.el9
-      - docker-ce: 3:27.2.0-1.el9
-      - docker-ce-cli: 1:27.2.0-1.el9
-      - docker-ce-rootless-extras: 27.2.0-1.el9
+      - containerd.io: 1.6.21-3.1.el9
+      - docker-ce: 24.0.4-1.el9
+      - docker-ce-cli: 24.0.4-1.el9
+      - docker-ce-rootless-extras: 24.0.4-1.el9
    - hold: True
    - update_holds: True
 {% endif %}
--- a/salt/docker/soc_docker.yaml
+++ b/salt/docker/soc_docker.yaml
@@ -45,7 +45,6 @@ docker:
    so-influxdb: *dockerOptions
    so-kibana: *dockerOptions
    so-kratos: *dockerOptions
-    so-hydra: *dockerOptions
    so-logstash: *dockerOptions
    so-nginx: *dockerOptions
    so-nginx-fleet-node: *dockerOptions
@@ -64,42 +63,6 @@ docker:
    so-elastic-agent: *dockerOptions
    so-telegraf: *dockerOptions
    so-steno: *dockerOptions
-    so-suricata:
-      final_octet:
-        description: Last octet of the container IP address.
-        helpLink: docker.html
-        readonly: True
-        advanced: True
-        global: True
-      port_bindings:
-        description: List of port bindings for the container.
-        helpLink: docker.html
-        advanced: True
-        multiline: True
-        forcedType: "[]string"
-      custom_bind_mounts:
-        description: List of custom local volume bindings.
-        advanced: True
-        helpLink: docker.html
-        multiline: True
-        forcedType: "[]string"
-      extra_hosts:
-        description: List of additional host entries for the container.
-        advanced: True
-        helpLink: docker.html
-        multiline: True
-        forcedType: "[]string"
-      extra_env:
-        description: List of additional ENV entries for the container.
-        advanced: True
-        helpLink: docker.html
-        multiline: True
-        forcedType: "[]string"
-      ulimits:
-        description: Ulimits for the container, in bytes.
-        advanced: True
-        helpLink: docker.html
-        multiline: True
-        forcedType: "[]string"
+    so-suricata: *dockerOptions
    so-zeek: *dockerOptions
    so-kafka: *dockerOptions
--- a/salt/elastalert/config.sls
+++ b/salt/elastalert/config.sls
@@ -82,36 +82,6 @@ elastasomodulesync:
    - group: 933
    - makedirs: True

-elastacustomdir:
-  file.directory:
-    - name: /opt/so/conf/elastalert/custom
-    - user: 933
-    - group: 933
-    - makedirs: True
-
-elastacustomsync:
-  file.recurse:
-    - name: /opt/so/conf/elastalert/custom
-    - source: salt://elastalert/files/custom
-    - user: 933
-    - group: 933
-    - makedirs: True
-    - file_mode: 660
-    - show_changes: False
-
-elastapredefinedsync:
-  file.recurse:
-    - name: /opt/so/conf/elastalert/predefined
-    - source: salt://elastalert/files/predefined
-    - user: 933
-    - group: 933
-    - makedirs: True
-    - template: jinja
-    - file_mode: 660
-    - context:
-        elastalert: {{ ELASTALERTMERGED }}
-    - show_changes: False
-
 elastaconf:
  file.managed:
    - name: /opt/so/conf/elastalert/elastalert_config.yaml
--- a/salt/elastalert/defaults.yaml
+++ b/salt/elastalert/defaults.yaml
@@ -1,6 +1,5 @@
 elastalert:
  enabled: False
-  alerter_parameters: ""
  config:
    rules_folder: /opt/elastalert/rules/
    scan_subdirectories: true
--- a/salt/elastalert/enabled.sls
+++ b/salt/elastalert/enabled.sls
@@ -30,8 +30,6 @@ so-elastalert:
      - /opt/so/rules/elastalert:/opt/elastalert/rules/:ro
      - /opt/so/log/elastalert:/var/log/elastalert:rw
      - /opt/so/conf/elastalert/modules/:/opt/elastalert/modules/:ro
-      - /opt/so/conf/elastalert/predefined/:/opt/elastalert/predefined/:ro
-      - /opt/so/conf/elastalert/custom/:/opt/elastalert/custom/:ro
      - /opt/so/conf/elastalert/elastalert_config.yaml:/opt/elastalert/config.yaml:ro
      {% if DOCKER.containers['so-elastalert'].custom_bind_mounts %}
        {% for BIND in DOCKER.containers['so-elastalert'].custom_bind_mounts %}
--- a/salt/elastalert/files/custom/placeholder
+++ b/salt/elastalert/files/custom/placeholder
@@ -1 +0,0 @@
-THIS IS A PLACEHOLDER FILE
--- a/salt/elastalert/files/modules/so/playbook-es.py
+++ b/salt/elastalert/files/modules/so/playbook-es.py
@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+from time import gmtime, strftime
+import requests,json
+from elastalert.alerts import Alerter
+
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+class PlaybookESAlerter(Alerter):
+    """
+    Use matched data to create alerts in elasticsearch
+    """
+
+    required_options = set(['play_title','play_url','sigma_level'])
+
+    def alert(self, matches):
+       for match in matches:
+            today = strftime("%Y.%m.%d", gmtime())
+            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
+            headers = {"Content-Type": "application/json"}
+
+            creds = None
+            if 'es_username' in self.rule and 'es_password' in self.rule:
+                creds = (self.rule['es_username'], self.rule['es_password'])
+
+            payload = {"tags":"alert","rule": { "name": self.rule['play_title'],"case_template": self.rule['play_id'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp}
+            url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-playbook.alerts-so/_doc/"
+            requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
+                            
+    def get_info(self):
+        return {'type': 'PlaybookESAlerter'} 
--- a/salt/elastalert/files/modules/so/securityonion-es.py
+++ b/salt/elastalert/files/modules/so/securityonion-es.py
@@ -1,63 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-from time import gmtime, strftime
-import requests,json
-from elastalert.alerts import Alerter
-
-import urllib3
-urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-
-class SecurityOnionESAlerter(Alerter):
-    """
-    Use matched data to create alerts in Elasticsearch.
-    """
-
-    required_options = set(['detection_title', 'sigma_level'])
-    optional_fields = ['sigma_category', 'sigma_product', 'sigma_service']
-
-    def alert(self, matches):
-        for match in matches:
-            timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S"'.000Z', gmtime())
-            headers = {"Content-Type": "application/json"}
-
-            creds = None
-            if 'es_username' in self.rule and 'es_password' in self.rule:
-                creds = (self.rule['es_username'], self.rule['es_password'])
-
-            # Start building the rule dict
-            rule_info = {
-                "name": self.rule['detection_title'],
-                "uuid": self.rule['detection_public_id']
-            }
-
-            # Add optional fields if they are present in the rule
-            for field in self.optional_fields:
-                rule_key = field.split('_')[-1]  # Assumes field format "sigma_<key>"
-                if field in self.rule:
-                    rule_info[rule_key] = self.rule[field]
-
-            # Construct the payload with the conditional rule_info
-            payload = {
-                "tags": "alert",
-                "rule": rule_info,
-                "event": {
-                    "severity": self.rule['event.severity'],
-                    "module": self.rule['event.module'],
-                    "dataset": self.rule['event.dataset'],
-                    "severity_label": self.rule['sigma_level']
-                },
-                "sigma_level": self.rule['sigma_level'],
-                "event_data": match,
-                "@timestamp": timestamp
-            }
-            url = f"https://{self.rule['es_host']}:{self.rule['es_port']}/logs-detections.alerts-so/_doc/"
-            requests.post(url, data=json.dumps(payload), headers=headers, verify=False, auth=creds)
-
-    def get_info(self):
-        return {'type': 'SecurityOnionESAlerter'}
--- a/salt/elastalert/files/predefined/jira_auth.yaml
+++ b/salt/elastalert/files/predefined/jira_auth.yaml
@@ -1,6 +0,0 @@
-{% if elastalert.get('jira_user', '') | length > 0 and elastalert.get('jira_pass', '') | length > 0 %}
-user: {{ elastalert.jira_user }}
-password: {{ elastalert.jira_pass }}
-{% else %}
-apikey: {{ elastalert.get('jira_api_key', '') }}
-{% endif %}
--- a/salt/elastalert/files/predefined/smtp_auth.yaml
+++ b/salt/elastalert/files/predefined/smtp_auth.yaml
@@ -1,2 +0,0 @@
-user: {{ elastalert.get('smtp_user', '') }}
-password: {{ elastalert.get('smtp_pass', '') }}
--- a/salt/elastalert/map.jinja
+++ b/salt/elastalert/map.jinja
@@ -13,19 +13,3 @@
 {% do ELASTALERTDEFAULTS.elastalert.config.update({'es_password': pillar.elasticsearch.auth.users.so_elastic_user.pass}) %}

 {% set ELASTALERTMERGED = salt['pillar.get']('elastalert', ELASTALERTDEFAULTS.elastalert, merge=True) %}
-
-{% if 'ntf' in salt['pillar.get']('features', []) %}
-   {% set params = ELASTALERTMERGED.get('alerter_parameters', '') | load_yaml %}
-   {% if params != None and params | length > 0 %}
-      {% do ELASTALERTMERGED.config.update(params) %}
-   {% endif %}
-
-   {% if ELASTALERTMERGED.get('smtp_user', '') | length > 0 %}
-      {% do ELASTALERTMERGED.config.update({'smtp_auth_file': '/opt/elastalert/predefined/smtp_auth.yaml'}) %}
-   {% endif %}
-
-   {% if ELASTALERTMERGED.get('jira_user', '') | length > 0 or ELASTALERTMERGED.get('jira_key', '') | length > 0 %}
-      {% do ELASTALERTMERGED.config.update({'jira_account_file': '/opt/elastalert/predefined/jira_auth.yaml'}) %}
-   {% endif %}
-
-{% endif %}
--- a/salt/elastalert/soc_elastalert.yaml
+++ b/salt/elastalert/soc_elastalert.yaml
@@ -1,100 +1,7 @@
 elastalert:
  enabled:
-    description: Enables or disables the ElastAlert 2 process. This process is critical for ensuring alerts arrive in SOC, and for outbound notification delivery.
+    description: You can enable or disable Elastalert.
    helpLink: elastalert.html
-  alerter_parameters:
-    title: Custom Configuration Parameters
-    description: Optional configuration parameters made available as defaults for all rules and alerters. Use YAML format for these parameters, and reference the ElastAlert 2 documentation, located at https://elastalert2.readthedocs.io, for available configuration parameters. Requires a valid Security Onion license key.
-    global: True
-    multiline: True
-    syntax: yaml
-    helpLink: elastalert.html
-    forcedType: string
-  jira_api_key:
-    title: Jira API Key
-    description: Optional configuration parameter for Jira API Key, used instead of the Jira username and password. Requires a valid Security Onion license key.
-    global: True
-    sensitive: True
-    helpLink: elastalert.html
-    forcedType: string
-  jira_pass:
-    title: Jira Password
-    description: Optional configuration parameter for Jira password. Requires a valid Security Onion license key.
-    global: True
-    sensitive: True
-    helpLink: elastalert.html
-    forcedType: string
-  jira_user:
-    title: Jira Username
-    description: Optional configuration parameter for Jira username. Requires a valid Security Onion license key.
-    global: True
-    helpLink: elastalert.html
-    forcedType: string
-  smtp_pass:
-    title: SMTP Password
-    description: Optional configuration parameter for SMTP password, required for authenticating email servers. Requires a valid Security Onion license key.
-    global: True
-    sensitive: True
-    helpLink: elastalert.html
-    forcedType: string
-  smtp_user:
-    title: SMTP Username
-    description: Optional configuration parameter for SMTP username, required for authenticating email servers. Requires a valid Security Onion license key.
-    global: True
-    helpLink: elastalert.html
-    forcedType: string
-  files:
-    custom:
-      alertmanager_ca__crt:
-        description: Optional custom Certificate Authority for connecting to an AlertManager server. To utilize this custom file, the alertmanager_ca_certs key must be set to /opt/elastalert/custom/alertmanager_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
-        global: True
-        file: True
-        helpLink: elastalert.html
-      gelf_ca__crt:
-        description: Optional custom Certificate Authority for connecting to a Graylog server. To utilize this custom file, the graylog_ca_certs key must be set to /opt/elastalert/custom/graylog_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
-        global: True
-        file: True
-        helpLink: elastalert.html
-      http_post_ca__crt:
-        description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the legacy HTTP POST alerter. To utilize this custom file, the http_post_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
-        global: True
-        file: True
-        helpLink: elastalert.html
-      http_post2_ca__crt:
-        description: Optional custom Certificate Authority for connecting to a generic HTTP server, via the newer HTTP POST 2 alerter. To utilize this custom file, the http_post2_ca_certs key must be set to /opt/elastalert/custom/http_post2_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
-        global: True
-        file: True
-        helpLink: elastalert.html
-      ms_teams_ca__crt:
-        description: Optional custom Certificate Authority for connecting to Microsoft Teams server. To utilize this custom file, the ms_teams_ca_certs key must be set to /opt/elastalert/custom/ms_teams_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
-        global: True
-        file: True
-        helpLink: elastalert.html
-      pagerduty_ca__crt:
-        description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the pagerduty_ca_certs key must be set to /opt/elastalert/custom/pagerduty_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
-        global: True
-        file: True
-        helpLink: elastalert.html
-      rocket_chat_ca__crt:
-        description: Optional custom Certificate Authority for connecting to PagerDuty server. To utilize this custom file, the rocket_chart_ca_certs key must be set to /opt/elastalert/custom/rocket_chat_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
-        global: True
-        file: True
-        helpLink: elastalert.html
-      smtp__crt:
-        description: Optional custom certificate for connecting to an SMTP server. To utilize this custom file, the smtp_cert_file key must be set to /opt/elastalert/custom/smtp.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
-        global: True
-        file: True
-        helpLink: elastalert.html
-      smtp__key:
-        description: Optional custom certificate key for connecting to an SMTP server. To utilize this custom file, the smtp_key_file key must be set to /opt/elastalert/custom/smtp.key in the Alerter Parameters setting. Requires a valid Security Onion license key.
-        global: True
-        file: True
-        helpLink: elastalert.html
-      slack_ca__crt:
-        description: Optional custom Certificate Authority for connecting to Slack. To utilize this custom file, the slack_ca_certs key must be set to /opt/elastalert/custom/slack_ca.crt in the Alerter Parameters setting. Requires a valid Security Onion license key.
-        global: True
-        file: True
-        helpLink: elastalert.html
  config:
    disable_rules_on_error:
      description: Disable rules on failure.
--- a/salt/elastic-fleet-package-registry/soc_elastic-fleet-package-registry.yaml
+++ b/salt/elastic-fleet-package-registry/soc_elastic-fleet-package-registry.yaml
@@ -1,4 +1,4 @@
 elastic_fleet_package_registry:
  enabled:
-    description: Enables or disables the Fleet package registry process. This process must remain enabled to allow Elastic Agent packages to be updated.
+    description: You can enable or disable Elastic Fleet Package Registry.
    advanced: True
--- a/salt/elasticagent/enabled.sls
+++ b/salt/elasticagent/enabled.sls
@@ -8,6 +8,7 @@
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}

+
 include:
  - elasticagent.config
  - elasticagent.sostatus
--- a/salt/elasticagent/soc_elasticagent.yaml
+++ b/salt/elasticagent/soc_elasticagent.yaml
@@ -1,4 +0,0 @@
-elasticagent:
-  enabled:
-    description: Enables or disables the Elastic Agent process. This process must remain enabled to allow collection of node events.
-    advanced: True
--- a/salt/elasticfleet/config.sls
+++ b/salt/elasticfleet/config.sls
@@ -63,14 +63,6 @@ eastatedir:
    - group: 939
    - makedirs: True

-custommappingsdir:
-  file.directory:
-    - name: /nsm/custom-mappings
-    - user: 947
-    - group: 939
-    - makedirs: True
-
-
 eapackageupgrade:
  file.managed:
    - name: /usr/sbin/so-elastic-fleet-package-upgrade
@@ -81,56 +73,6 @@ eapackageupgrade:
    - template: jinja

 {%   if GLOBALS.role != "so-fleet" %}
-   
-{% if not GLOBALS.airgap %}
-soresourcesrepoclone:
-  git.latest:
-    - name: https://github.com/Security-Onion-Solutions/securityonion-resources.git
-    - target: /nsm/securityonion-resources
-    - rev: 'main'
-    - depth: 1
-    - force_reset: True
-{% endif %}
-
-elasticdefendconfdir:
-  file.directory:
-    - name: /opt/so/conf/elastic-fleet/defend-exclusions/rulesets
-    - user: 947
-    - group: 939
-    - makedirs: True
-  
-elasticdefenddisabled:
-  file.managed:
-    - name: /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml
-    - source: salt://elasticfleet/files/soc/elastic-defend-disabled-filters.yaml
-    - user: 947
-    - group: 939
-    - mode: 600
-
-elasticdefendcustom:
-  file.managed:
-    - name: /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters-raw
-    - source: salt://elasticfleet/files/soc/elastic-defend-custom-filters.yaml
-    - user: 947
-    - group: 939
-    - mode: 600
-
-{% if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
-{%   set ap = "present" %}
-{% else %}
-{%   set ap = "absent" %}
-{% endif %}
-cron-elastic-defend-filters:
-  cron.{{ap}}:
-    - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
-    - identifier: elastic-defend-filters
-    - user: root
-    - minute: '0'
-    - hour: '3'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-
 eaintegrationsdir:
  file.directory:
    - name: /opt/so/conf/elastic-fleet/integrations
--- a/salt/elasticfleet/defaults.yaml
+++ b/salt/elasticfleet/defaults.yaml
@@ -8,8 +8,6 @@ elasticfleet:
      endpoints_enrollment: ''
      es_token: ''
      grid_enrollment: ''
-    defend_filters:
-      enable_auto_configuration: False
  logging:
    zeek:
      excluded:
@@ -38,9 +36,7 @@ elasticfleet:
    - aws
    - azure
    - barracuda
-    - barracuda_cloudgen_firewall
    - carbonblack_edr
-    - cef
    - checkpoint
    - cisco_asa
    - cisco_duo
@@ -48,12 +44,10 @@ elasticfleet:
    - cisco_ios
    - cisco_ise
    - cisco_meraki
-    - cisco_secure_email_gateway
    - cisco_umbrella
    - citrix_adc
    - citrix_waf
    - cloudflare
-    - cloudflare_logpush
    - crowdstrike
    - darktrace
    - elastic_agent
@@ -71,7 +65,6 @@ elasticfleet:
    - http_endpoint
    - httpjson
    - iis
-    - imperva_cloud_waf
    - journald
    - juniper
    - juniper_srx
@@ -103,19 +96,14 @@ elasticfleet:
    - symantec_endpoint
    - system
    - tcp
-    - tenable_io
    - tenable_sc
    - ti_abusech
    - ti_anomali
    - ti_cybersixgill
    - ti_misp
-    - ti_opencti
    - ti_otx
-    - ti_rapid7_threat_command
    - ti_recordedfuture
    - ti_threatq
-    - trendmicro
-    - trend_micro_vision_one
    - udp
    - vsphere
    - windows
@@ -130,8 +118,3 @@ elasticfleet:
      base_url: https://api.platform.sublimesecurity.com
      poll_interval: 5m
      limit: 100
-    kismet:
-      base_url: http://localhost:2501
-      poll_interval: 1m
-      api_key:
-      enabled_nodes: []
--- a/salt/elasticfleet/enabled.sls
+++ b/salt/elasticfleet/enabled.sls
@@ -17,21 +17,17 @@ include:
  - elasticfleet.sostatus
  - ssl

-{% if grains.role not in ['so-fleet'] %}
 # Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready
 wait_for_elasticsearch_elasticfleet:
  cmd.run:
    - name: so-elasticsearch-wait
-{% endif %}

 # If enabled, automatically update Fleet Logstash Outputs
 {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %}
 so-elastic-fleet-auto-configure-logstash-outputs:
  cmd.run:
    - name: /usr/sbin/so-elastic-fleet-outputs-update
-    - retry:
-        attempts: 4
-        interval: 30
+    - retry: True
 {% endif %}

 # If enabled, automatically update Fleet Server URLs & ES Connection
@@ -39,9 +35,7 @@ so-elastic-fleet-auto-configure-logstash-outputs:
 so-elastic-fleet-auto-configure-server-urls:
  cmd.run:
    - name: /usr/sbin/so-elastic-fleet-urls-update
-    - retry:
-        attempts: 4
-        interval: 30
+    - retry: True
 {% endif %}

 # Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
@@ -49,16 +43,12 @@ so-elastic-fleet-auto-configure-server-urls:
 so-elastic-fleet-auto-configure-elasticsearch-urls:
  cmd.run:
    - name: /usr/sbin/so-elastic-fleet-es-url-update
-    - retry:
-        attempts: 4
-        interval: 30
+    - retry: True

 so-elastic-fleet-auto-configure-artifact-urls:
  cmd.run:
    - name: /usr/sbin/so-elastic-fleet-artifacts-url-update
-    - retry:
-        attempts: 4
-        interval: 30
+    - retry: True 

 {% endif %}

@@ -143,22 +133,7 @@ so-elastic-fleet-integrations:
 so-elastic-agent-grid-upgrade:
  cmd.run:
    - name: /usr/sbin/so-elastic-agent-grid-upgrade
-    - retry:
-        attempts: 12
-        interval: 5
-
-so-elastic-fleet-integration-upgrade:
-  cmd.run:
-    - name: /usr/sbin/so-elastic-fleet-integration-upgrade
-
-{%   if ELASTICFLEETMERGED.config.defend_filters.enable_auto_configuration %}
-so-elastic-defend-manage-filters-file-watch:
-  cmd.run:
-    - name: python3 /sbin/so-elastic-defend-manage-filters.py -c /opt/so/conf/elasticsearch/curl.config -d /opt/so/conf/elastic-fleet/defend-exclusions/disabled-filters.yaml -i /nsm/securityonion-resources/event_filters/ -i /opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters/ &>> /opt/so/log/elasticfleet/elastic-defend-manage-filters.log
-    - onchanges:
-      - file: elasticdefendcustom
-      - file: elasticdefenddisabled
-{%    endif %}
+    - retry: True
 {%  endif %}

 delete_so-elastic-fleet_so-status.disabled:
--- a/salt/elasticfleet/files/integrations-dynamic/fleet-server/fleet-server.json
+++ b/salt/elasticfleet/files/integrations-dynamic/fleet-server/fleet-server.json
@@ -1,19 +0,0 @@
-{
-  "package": {
-    "name": "fleet_server",
-    "version": ""
-  },
-  "name": "fleet_server-1",
-  "namespace": "default",
-  "policy_id": "FleetServer_hostname",
-  "vars": {},
-  "inputs": {
-    "fleet_server-fleet-server": {
-      "enabled": true,
-      "vars": {
-        "custom": "server.ssl.supported_protocols: [\"TLSv1.2\", \"TLSv1.3\"]\nserver.ssl.cipher_suites: [ \"ECDHE-RSA-AES-128-GCM-SHA256\", \"ECDHE-RSA-AES-256-GCM-SHA384\", \"ECDHE-RSA-AES-128-CBC-SHA\", \"ECDHE-RSA-AES-256-CBC-SHA\", \"RSA-AES-128-GCM-SHA256\", \"RSA-AES-256-GCM-SHA384\"]"
-      },
-      "streams": {}
-    }
-  }
-}
--- a/salt/elasticfleet/files/integrations-optional/kismet.json
+++ b/salt/elasticfleet/files/integrations-optional/kismet.json
@@ -1,36 +0,0 @@
-{% from 'elasticfleet/map.jinja' import ELASTICFLEETMERGED %}
-{% raw %}
-{
-  "package": {
-    "name": "httpjson",
-    "version": ""
-  },
-  "name": "kismet-logs",
-  "namespace": "so",
-  "description": "Kismet Logs",
-  "policy_id": "FleetServer_{% endraw %}{{ NAME }}{% raw %}",
-  "inputs": {
-    "generic-httpjson": {
-      "enabled": true,
-      "streams": {
-        "httpjson.generic": {
-          "enabled": true,
-          "vars": {
-            "data_stream.dataset": "kismet",
-            "request_url": "{% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.base_url }}{% raw %}/devices/last-time/-600/devices.tjson",
-            "request_interval": "{% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.poll_interval }}{% raw %}",
-            "request_method": "GET",
-            "request_transforms": "- set:\r\n    target: header.Cookie\r\n    value: 'KISMET={% endraw %}{{ ELASTICFLEETMERGED.optional_integrations.kismet.api_key }}{% raw %}'",
-            "request_redirect_headers_ban_list": [],
-            "oauth_scopes": [],
-            "processors": "",
-            "tags": [],
-            "pipeline": "kismet.common"
-          }
-        }
-      }
-    }
-  },
-  "force": true
-}
-{% endraw %}
--- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json
+++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json
@@ -5,7 +5,7 @@
 	"package": {
 		"name": "endpoint",
 		"title": "Elastic Defend",
-		"version": "8.14.0"
+		"version": "8.10.2"
 	},
 	"enabled": true,
 	"policy_id": "endpoints-initial",
--- a/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json
+++ b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json
@@ -11,7 +11,7 @@
    "winlogs-winlog": {
      "enabled": true,
      "streams": {
-        "winlog.winlogs": {
+        "winlog.winlog": {
          "enabled": true,
          "vars": {
            "channel": "Microsoft-Windows-Windows Defender/Operational",
--- a/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json
+++ b/salt/elasticfleet/files/integrations/grid-nodes_general/hydra-logs.json
@@ -1,30 +0,0 @@
-{
-  "package": {
-    "name": "log",
-    "version": ""
-  },
-  "name": "hydra-logs",
-  "namespace": "so",
-  "description": "Hydra logs",
-  "policy_id": "so-grid-nodes_general",
-  "inputs": {
-    "logs-logfile": {
-      "enabled": true,
-      "streams": {
-        "log.logs": {
-          "enabled": true,
-          "vars": {
-            "paths": [
-              "/opt/so/log/hydra/hydra.log"
-            ],
-            "data_stream.dataset": "hydra",
-            "tags": ["so-hydra"],
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: hydra",
-            "custom": "pipeline: hydra"
-          }
-        }
-      }
-    }
-  },
-  "force": true
-}
--- a/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json
+++ b/salt/elasticfleet/files/integrations/grid-nodes_general/import-evtx-logs.json
@@ -20,7 +20,7 @@
            ],
            "data_stream.dataset": "import",
            "custom": "",
-            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-1.59.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-1.45.1\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-1.59.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-1.59.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-1.45.1\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
+            "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- drop_fields:\n    fields: [\"host\"]\n    ignore_missing: true\n- add_fields:\n    target: data_stream\n    fields:\n      type: logs\n      dataset: system.security\n- add_fields:\n    target: event\n    fields:\n      dataset: system.security\n      module: system\n      imported: true\n- add_fields:\n    target: \"@metadata\"\n    fields:\n        pipeline: logs-system.security-1.43.0\n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-Sysmon/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.sysmon_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.sysmon_operational\n          module: windows\n          imported: true\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.sysmon_operational-1.38.0\n- if:\n    equals:\n      winlog.channel: 'Application'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.application\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.application-1.43.0\n- if:\n    equals:\n      winlog.channel: 'System'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: event\n        fields:\n          dataset: system.system\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-system.system-1.43.0\n    \n- if:\n    equals:\n      winlog.channel: 'Microsoft-Windows-PowerShell/Operational'\n  then: \n    - add_fields:\n        target: data_stream\n        fields:\n          dataset: windows.powershell_operational\n    - add_fields:\n        target: event\n        fields:\n          dataset: windows.powershell_operational\n          module: windows\n    - add_fields:\n        target: \"@metadata\"\n        fields:\n            pipeline: logs-windows.powershell_operational-1.38.0\n- add_fields:\n    target: data_stream\n    fields:\n        dataset: import",
            "tags": [
              "import"
            ]
--- a/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json
+++ b/salt/elasticfleet/files/integrations/grid-nodes_general/so-ip-mappings.json
@@ -1,35 +0,0 @@
-{
-  "package": {
-    "name": "log",
-    "version": ""
-  },
-  "name": "so-ip-mappings",
-  "namespace": "so",
-  "description": "IP Description mappings",
-  "policy_id": "so-grid-nodes_general",
-  "vars": {},
-  "inputs": {
-    "logs-logfile": {
-      "enabled": true,
-      "streams": {
-        "log.logs": {
-          "enabled": true,
-          "vars": {
-            "paths": [
-              "/nsm/custom-mappings/ip-descriptions.csv"
-            ],
-            "data_stream.dataset": "hostnamemappings",
-            "tags": [
-              "so-ip-mappings"
-            ],
-            "processors": "- decode_csv_fields:\n    fields:\n      message: decoded.csv\n    separator: \",\"\n    ignore_missing: false\n    overwrite_keys: true\n    trim_leading_space: true\n    fail_on_error: true\n\n- extract_array:\n    field: decoded.csv\n    mappings:\n      so.ip_address: '0'\n      so.description: '1'\n\n- script:\n    lang: javascript\n    source: >\n      function process(event) {\n          var ip = event.Get('so.ip_address');\n          var validIpRegex = /^((25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)\\.){3}(25[0-5]|2[0-4]\\d|1\\d{2}|[1-9]?\\d)$/\n          if (!validIpRegex.test(ip)) {\n              event.Cancel();\n          }\n      }\n- fingerprint:\n    fields: [\"so.ip_address\"]\n    target_field: \"@metadata._id\"\n",
-            "custom": ""
-          }
-        }
-      }
-    }
-  },
-  "force": true
-}
-
-
--- a/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json
+++ b/salt/elasticfleet/files/integrations/grid-nodes_general/soc-detections-logs.json
@@ -1,35 +0,0 @@
-{
-  "policy_id": "so-grid-nodes_general",
-  "package": {
-    "name": "log",
-    "version": ""
-  },
-  "name": "soc-detections-logs",
-  "description": "Security Onion Console - Detections Logs",
-  "namespace": "so",
-  "inputs": {
-    "logs-logfile": {
-      "enabled": true,
-      "streams": {
-        "log.logs": {
-          "enabled": true,
-          "vars": {
-            "paths": [
-              "/opt/so/log/soc/detections_runtime-status_sigma.log",
-              "/opt/so/log/soc/detections_runtime-status_yara.log"
-            ],
-            "exclude_files": [],
-            "ignore_older": "72h",
-            "data_stream.dataset": "soc",
-            "tags": [
-              "so-soc"
-            ],
-            "processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: detections\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true",
-            "custom": "pipeline: common"
-          }
-        }
-      }
-    }
-  },
-  "force": true
-}
--- a/salt/elasticfleet/files/soc/elastic-defend-custom-filters.yaml
+++ b/salt/elasticfleet/files/soc/elastic-defend-custom-filters.yaml
@@ -1,27 +0,0 @@
-title: 'Template 1'
-id: 'This needs to be a UUIDv4 id - https://www.uuidgenerator.net/version4'
-description: 'Short description detailing what this rule is filtering and why.'
-references: 'Relevant urls, etc'
-author: '@SecurityOnion'
-date: 'MM/DD/YY'
-event_type: 'dns_query'
-filter_type: 'exclude'
-filter:
-  selection_1:
-    TargetField: 'QueryName'
-    Condition: 'end with'
-    Pattern: '.thawte.com'
---
-title: 'Template 2'
-id: 'This needs to be a UUIDv4 id - https://www.uuidgenerator.net/version4'
-description: 'Short description detailing what this rule is filtering and why.'
-references: 'Relevant urls, etc'
-author: '@SecurityOnion'
-date: 'MM/DD/YY'
-event_type: 'process_creation'
-filter_type: 'exclude'
-filter:
-  selection_1:
-    TargetField: 'ParentImage'
-    Condition: 'is'
-    Pattern: 'C:\Windows\Microsoft.NET\Framework\v4.0.30319\ngentask.exe'
--- a/salt/elasticfleet/files/soc/elastic-defend-disabled-filters.yaml
+++ b/salt/elasticfleet/files/soc/elastic-defend-disabled-filters.yaml
@@ -1,3 +0,0 @@
-'9EDAA51C-BB12-49D9-8748-2B61371F2E7D':
-  Date: '10/10/2024'
-  Notes: 'Example Disabled Filter - Leave this entry here, just copy and paste as needed.'
--- a/salt/elasticfleet/soc_elasticfleet.yaml
+++ b/salt/elasticfleet/soc_elasticfleet.yaml
@@ -1,6 +1,6 @@
 elasticfleet:
  enabled:
-    description: Enables or disables the Elastic Fleet process. This process is critical for managing Elastic Agents.
+    description: You can enable or disable Elastic Fleet.
    advanced: True
    helpLink: elastic-fleet.html
  enable_manager_output:
@@ -9,24 +9,6 @@ elasticfleet:
    global: True
    forcedType: bool
    helpLink: elastic-fleet.html
-  files:
-    soc:
-      elastic-defend-disabled-filters__yaml:
-        title: Disabled Elastic Defend filters
-        description: Enter the ID of the filter that should be disabled.
-        syntax: yaml
-        file: True
-        global: True
-        helpLink: elastic-fleet.html
-        advanced: True
-      elastic-defend-custom-filters__yaml:
-        title: Custom Elastic Defend filters
-        description: Enter custom filters seperated by ---
-        syntax: yaml
-        file: True
-        global: True
-        helpLink: elastic-fleet.html
-        advanced: True
  logging:
    zeek:
      excluded:
@@ -34,12 +16,6 @@ elasticfleet:
        forcedType: "[]string"
        helpLink: zeek.html
  config:
-    defend_filters:
-      enable_auto_configuration:
-        description: Enable auto-configuration and management of the Elastic Defend Exclusion filters.
-        global: True
-        helpLink: elastic-fleet.html
-        advanced: True
    server:
      custom_fqdn:
        description: Custom FQDN for Agents to connect to. One per line.
@@ -103,29 +79,3 @@ elasticfleet:
        helpLink: elastic-fleet.html
        advanced: True
        forcedType: int
-    kismet:
-      base_url:
-        description: Base URL for Kismet.
-        global: True
-        helpLink: elastic-fleet.html
-        advanced: True
-        forcedType: string
-      poll_interval:
-        description: Poll interval for wireless device data from Kismet. Integration is currently configured to return devices seen as active by any Kismet sensor within the last 10 minutes.
-        global: True
-        helpLink: elastic-fleet.html
-        advanced: True
-        forcedType: string
-      api_key:
-        description: API key for Kismet.
-        global: True
-        helpLink: elastic-fleet.html
-        advanced: True
-        forcedType: string
-        sensitive: True
-      enabled_nodes:
-        description: Fleet nodes with the Kismet integration enabled. Enter one per line.
-        global: True
-        helpLink: elastic-fleet.html
-        advanced: True
-        forcedType: "[]string"
--- a/salt/elasticfleet/tools/sbin/so-elastic-defend-manage-filters.py
+++ b/salt/elasticfleet/tools/sbin/so-elastic-defend-manage-filters.py
@@ -1,251 +0,0 @@
-from datetime import datetime
-import sys
-import getopt
-from so_elastic_defend_filters_helper import *
-import logging
-
-logging.basicConfig(level=logging.INFO, format='%(message)s')
-
-# Define mappings for Target Field, Event Type, Conditions
-TARGET_FIELD_MAPPINGS = {
-    "Image": "process.executable",
-    "ParentImage": "process.parent.executable",
-    "CommandLine": "process.command_line",
-    "ParentCommandLine": "process.parent.command_line",
-    "DestinationHostname": "destination.domain",
-    "QueryName": "dns.question.name",
-    "DestinationIp": "destination.ip",
-    "TargetObject": "registry.path",
-    "TargetFilename": "file.path"
-}
-
-DATASET_MAPPINGS = {
-    "process_create": "endpoint.events.process",
-    "network_connection": "endpoint.events.network",
-    "file_create": "endpoint.events.file",
-    "file_delete": "endpoint.events.file",
-    "registry_event": "endpoint.events.registry",
-    "dns_query": "endpoint.events.network"
-}
-
-CONDITION_MAPPINGS = {
-    "is": ("included", "match"),
-    "end with": ("included", "wildcard"),
-    "begin with": ("included", "wildcard"),
-    "contains": ("included", "wildcard")
-}
-
-# Extract entries for a rule
-def extract_entries(data, event_type):
-    entries = []
-    filter_data = data.get('filter', {})
-    for value in filter_data.values():
-        target_field = TARGET_FIELD_MAPPINGS.get(value.get('TargetField', ''))
-        condition = value.get('Condition', '')
-        pattern = value.get('Pattern', '')
-
-        if condition not in CONDITION_MAPPINGS:
-            logging.error(f"Invalid condition: {condition}")
-
-        # Modify the pattern based on the condition
-        pattern = modify_pattern(condition, pattern)
-
-        operator, match_type = CONDITION_MAPPINGS[condition]
-
-        entries.append({
-            "field": target_field,
-            "operator": operator,
-            "type": match_type,
-            "value": pattern
-        })
-
-    # Add the event.dataset entry from DATASET_MAPPINGS
-    dataset_value = DATASET_MAPPINGS.get(event_type, '')
-    if dataset_value:
-        entries.append({
-            "field": "event.dataset",
-            "operator": "included",
-            "type": "match",
-            "value": dataset_value
-        })
-    else:
-        logging.error(f"No dataset mapping found for event_type: {event_type}")
-
-    return entries
-
-# Build the JSON
-def build_json_entry(entries, guid, event_type, context):
-    return {
-        "comments": [],
-        "entries": entries,
-        "item_id": guid,
-        "name": f"SO - {event_type} - {guid}",
-        "description": f"{context}\n\n  <<- Note: This filter is managed by Security Onion. ->>",
-        "namespace_type": "agnostic",
-        "tags": ["policy:all"],
-        "type": "simple",
-        "os_types": ["windows"],
-        "entries": entries
-    }
-
-# Check to see if the rule is disabled
-# If it is, make sure it is not active
-def disable_check(guid, disabled_rules, username, password):
-    if guid in disabled_rules:
-        logging.info(f"Rule {guid} is in the disabled rules list, confirming that is is actually disabled...")
-        existing_rule = api_request("GET", guid, username, password)
-
-        if existing_rule:
-            if api_request("DELETE", guid, username, password):
-                logging.info(f"Successfully deleted rule {guid}")
-                return True, "deleted"
-            else:
-                logging.error(f"Error deleting rule {guid}.")
-                return True, "Error deleting"
-        return True, "NOP"
-    return False, None
-
-def modify_pattern(condition, pattern):
-    """
-    Modify the pattern based on the condition.
-    - 'end with': Add '*' to the beginning of the pattern.
-    - 'begin with': Add '*' to the end of the pattern.
-    - 'contains': Add '*' to both the beginning and end of the pattern.
-    """
-    if isinstance(pattern, list):
-        # Apply modification to each pattern in the list if it's a list of patterns
-        return [modify_pattern(condition, p) for p in pattern]
-    
-    if condition == "end with":
-        return f"*{pattern}"
-    elif condition == "begin with":
-        return f"{pattern}*"
-    elif condition == "contains":
-        return f"*{pattern}*"
-    return pattern
-
-
-def process_rule_update_or_create(guid, json_entry, username, password):
-    existing_rule = api_request("GET", guid, username, password)
-
-    if existing_rule:
-        existing_rule_data = extract_relevant_fields(existing_rule)
-        new_rule_data = extract_relevant_fields(json_entry)
-        if generate_hash(existing_rule_data) != generate_hash(new_rule_data):
-            logging.info(f"Updating rule {guid}")
-            json_entry.pop("list_id", None)
-            api_request("PUT", guid, username, password, json_data=json_entry)
-            return "updated"
-        logging.info(f"Rule {guid} is up to date.")
-        return "no_change"
-    else:
-        logging.info(f"Creating new rule {guid}")
-        json_entry["list_id"] = "endpoint_event_filters"
-        api_request("POST", guid, username, password, json_data=json_entry)
-        return "new"
-
-# Main function for processing rules
-def process_rules(yaml_files, disabled_rules, username, password):
-    stats = {"rule_count": 0, "new": 0, "updated": 0, "no_change": 0, "disabled": 0, "deleted": 0}
-    for data in yaml_files:
-        logging.info(f"Processing rule: {data.get('id', '')}")
-        event_type = data.get('event_type', '')
-        guid = data.get('id', '')
-        dataset = DATASET_MAPPINGS.get(event_type, '')
-        context = data.get('description', '')
-
-        rule_deleted, state = disable_check(guid, disabled_rules, username, password)
-        if rule_deleted:
-            stats["disabled"] += 1
-            if state == "deleted":
-                stats["deleted"] += 1
-            continue
-
-        # Extract entries and build JSON
-        entries = extract_entries(data, event_type)
-        json_entry = build_json_entry(entries, guid, event_type, context)
-
-        # Process rule creation or update
-        status = process_rule_update_or_create(guid, json_entry, username, password)
-
-        stats[status] += 1
-        stats["rule_count"] += 1
-    return stats
-
-def parse_args(argv):
-    try:
-        opts, args = getopt.getopt(argv, "i:d:c:f:", ["input=", "disabled=", "credentials=", "flags_file="])
-    except getopt.GetoptError:
-        print("Usage: python so-elastic-defend-manage-filters.py -c <credentials_file> -d <disabled_file> -i <folder_of_yaml_files> [-f <flags_file>]")
-        sys.exit(2)
-    return opts
-
-def load_flags(file_path):
-    with open(file_path, 'r') as flags_file:
-        return flags_file.read().splitlines()
-
-def validate_inputs(credentials_file, disabled_file, yaml_directories):
-    if not credentials_file or not disabled_file or not yaml_directories:
-        print("Usage: python so-elastic-defend-manage-filters.py -c <credentials_file> -d <disabled_file> -i <folder_of_yaml_files> [-f <flags_file>]")
-        sys.exit(2)
-
-def main(argv):
-    credentials_file = ""
-    disabled_file = ""
-    yaml_directories = []
-
-    opts = parse_args(argv)
-
-    for opt, arg in opts:
-        if opt in ("-c", "--credentials"):
-            credentials_file = arg
-        elif opt in ("-d", "--disabled"):
-            disabled_file = arg
-        elif opt in ("-i", "--input"):
-            yaml_directories.append(arg)
-        elif opt in ("-f", "--flags_file"):
-            flags = load_flags(arg)
-            return main(argv + flags)
-
-    timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-    logging.info(f"\n{timestamp}")
-
-    validate_inputs(credentials_file, disabled_file, yaml_directories)
-
-    credentials = load_credentials(credentials_file)
-    if not credentials:
-        raise Exception("Failed to load credentials")
-
-    username, password = extract_auth_details(credentials)
-    if not username or not password:
-        raise Exception("Invalid credentials format")
-
-    custom_rules_input = '/opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters-raw'
-    custom_rules_output = '/opt/so/conf/elastic-fleet/defend-exclusions/rulesets/custom-filters'
-    prepare_custom_rules(custom_rules_input, custom_rules_output)
-    disabled_rules = load_disabled(disabled_file)
-
-    total_stats = {"rule_count": 0, "new": 0, "updated": 0, "no_change": 0, "disabled": 0, "deleted": 0}
-
-    for yaml_dir in yaml_directories:
-        yaml_files = load_yaml_files(yaml_dir)
-        stats = process_rules(yaml_files, disabled_rules, username, password)
-
-        for key in total_stats:
-            total_stats[key] += stats[key]
-
-    logging.info(f"\nProcessing Summary")
-    logging.info(f" - Total processed rules: {total_stats['rule_count']}")
-    logging.info(f" - New rules: {total_stats['new']}")
-    logging.info(f" - Updated rules: {total_stats['updated']}")
-    logging.info(f" - Disabled rules: {total_stats['deleted']}")
-    logging.info(f" - Rules with no changes: {total_stats['no_change']}")
-    logging.info(f"Rule status Summary")
-    logging.info(f" - Active rules: {total_stats['rule_count'] - total_stats['disabled']}")
-    logging.info(f" - Disabled rules: {total_stats['disabled']}")
-    timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-    logging.info(f"Execution completed at: {timestamp}")
-
-
-if __name__ == "__main__":
-    main(sys.argv[1:])
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-common
@@ -102,70 +102,6 @@ elastic_fleet_package_is_installed() {
    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET -H 'kbn-xsrf: true' "localhost:5601/api/fleet/epm/packages/$PACKAGE" | jq -r '.item.status'
 }

-elastic_fleet_agent_policy_ids() {
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].id
-    if [ $? -ne 0 ]; then
-        echo "Error: Failed to retrieve agent policies."
-        exit 1
-    fi
-}
-
-elastic_fleet_agent_policy_names() {
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies" | jq -r .items[].name
-    if [ $? -ne 0 ]; then
-        echo "Error: Failed to retrieve agent policies."
-        exit 1
-    fi
-}
-
-elastic_fleet_integration_policy_names() {
-    AGENT_POLICY=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r .item.package_policies[].name
-    if [ $? -ne 0 ]; then
-        echo "Error: Failed to retrieve integrations for '$AGENT_POLICY'."
-        exit 1
-    fi
-}
-
-elastic_fleet_integration_policy_package_name() {
-    AGENT_POLICY=$1
-    INTEGRATION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.name'
-    if [ $? -ne 0 ]; then
-        echo "Error: Failed to retrieve package name for '$INTEGRATION' in '$AGENT_POLICY'."
-        exit 1
-    fi
-}
-
-elastic_fleet_integration_policy_package_version() {
-    AGENT_POLICY=$1
-    INTEGRATION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .package.version'
-    if [ $? -ne 0 ]; then
-        echo "Error: Failed to retrieve package version for '$INTEGRATION' in '$AGENT_POLICY'."
-        exit 1
-    fi
-}
-
-elastic_fleet_integration_id() {
-    AGENT_POLICY=$1
-    INTEGRATION=$2
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X GET "localhost:5601/api/fleet/agent_policies/$AGENT_POLICY" | jq -r --arg INTEGRATION "$INTEGRATION" '.item.package_policies[] | select(.name==$INTEGRATION)| .id'
-    if [ $? -ne 0 ]; then
-        echo "Error: Failed to retrieve integration ID for '$INTEGRATION' in '$AGENT_POLICY'."
-        exit 1
-    fi
-}
-
-elastic_fleet_integration_policy_dryrun_upgrade() {
-    INTEGRATION_ID=$1
-    curl -s -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -H "Content-Type: application/json" -H 'kbn-xsrf: true' -L -X POST "localhost:5601/api/fleet/package_policies/upgrade/dryrun" -d "{\"packagePolicyIds\":[\"$INTEGRATION_ID\"]}"
-    if [ $? -ne 0 ]; then
-        echo "Error: Failed to complete dry run for '$INTEGRATION_ID'."
-        exit 1
-    fi
-}
-
 elastic_fleet_policy_create() {

    NAME=$1
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
@@ -1,29 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-elastic-fleet-common
-
-# Get all the fleet policies
-json_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -L -X GET "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true')
-
-# Extract the IDs that start with "FleetServer_"
-POLICY=$(echo "$json_output" | jq -r '.items[] | select(.id | startswith("FleetServer_")) | .id')
-
-# Iterate over each ID in the POLICY variable
-for POLICYNAME in $POLICY; do
-    printf "\nUpdating Policy: $POLICYNAME\n"
-
-    # First get the Integration ID
-    INTEGRATION_ID=$(/usr/sbin/so-elastic-fleet-agent-policy-view "$POLICYNAME" | jq -r '.item.package_policies[] | select(.package.name == "fleet_server") | .id')  
-
-    # Modify the default integration policy to update the policy_id and an with the correct naming
-	UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "$POLICYNAME" --arg name "fleet_server-$POLICYNAME" '
-    .policy_id = $policy_id |
-    .name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
-
-    # Now update the integration policy using the modified JSON
-    elastic_fleet_integration_update "$INTEGRATION_ID" "$UPDATED_INTEGRATION_POLICY"
-done
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-policy-load
@@ -12,10 +12,7 @@ if [ ! -f /opt/so/state/eaintegrations.txt ]; then
  # First, check for any package upgrades
  /usr/sbin/so-elastic-fleet-package-upgrade

-  # Second, update Fleet Server policies
-  /sbin/so-elastic-fleet-integration-policy-elastic-fleet-server
-
-  # Third, configure Elastic Defend Integration seperately
+  # Second, configure Elastic Defend Integration seperately
  /usr/sbin/so-elastic-fleet-integration-policy-elastic-defend

  # Initial Endpoints
--- a/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade
+++ b/salt/elasticfleet/tools/sbin/so-elastic-fleet-integration-upgrade
@@ -1,62 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-elastic-fleet-common
-
-curl_output=$(curl -s -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/)
-if [ $? -ne 0 ]; then
-    echo "Error: Failed to connect to Kibana."
-    exit 1
-fi
-
-IFS=$'\n'
-agent_policies=$(elastic_fleet_agent_policy_ids)
-if [ $? -ne 0 ]; then
-    echo "Error: Failed to retrieve agent policies."
-    exit 1
-fi
-
-for AGENT_POLICY in $agent_policies; do
-    integrations=$(elastic_fleet_integration_policy_names "$AGENT_POLICY")
-    for INTEGRATION in $integrations; do
-        if ! [[ "$INTEGRATION" == "elastic-defend-endpoints" ]] && ! [[ "$INTEGRATION" == "fleet_server-"* ]]; then
-            # Get package name so we know what package to look for when checking the current and latest available version
-            PACKAGE_NAME=$(elastic_fleet_integration_policy_package_name "$AGENT_POLICY" "$INTEGRATION")
-
-            # Get currently installed version of package
-            PACKAGE_VERSION=$(elastic_fleet_integration_policy_package_version "$AGENT_POLICY" "$INTEGRATION")
-
-            # Get latest available version of package
-            AVAILABLE_VERSION=$(elastic_fleet_package_latest_version_check "$PACKAGE_NAME")
-
-            # Get integration ID
-            INTEGRATION_ID=$(elastic_fleet_integration_id "$AGENT_POLICY" "$INTEGRATION")
-
-            if [[ "$PACKAGE_VERSION" != "$AVAILABLE_VERSION" ]]; then
-                # Dry run of the upgrade
-                echo "Current $PACKAGE_NAME package version ($PACKAGE_VERSION) is not the same as the latest available package ($AVAILABLE_VERSION)..."
-                echo "Upgrading $INTEGRATION..."
-                echo "Starting dry run..."
-                DRYRUN_OUTPUT=$(elastic_fleet_integration_policy_dryrun_upgrade "$INTEGRATION_ID")
-                DRYRUN_ERRORS=$(echo "$DRYRUN_OUTPUT" | jq .[].hasErrors)
-                
-                # If no errors with dry run, proceed with actual upgrade
-                if [[ "$DRYRUN_ERRORS" == "false" ]]; then
-                    echo "No errors detected. Proceeding with upgrade..."
-                    elastic_fleet_integration_policy_upgrade "$INTEGRATION_ID"
-                    if [ $? -ne 0 ]; then
-                        echo "Error: Upgrade failed for integration ID '$INTEGRATION_ID'."
-                        exit 1
-                    fi
-                else
-                    echo "Errors detected during dry run. Stopping upgrade..."
-                    exit 1
-                fi
-            fi
-        fi
-    done
-done
-echo
--- a/salt/elasticfleet/tools/sbin/so_elastic_defend_filters_helper.py
+++ b/salt/elasticfleet/tools/sbin/so_elastic_defend_filters_helper.py
@@ -1,128 +0,0 @@
-
-import hashlib
-import os
-import json
-import yaml
-import requests
-from requests.auth import HTTPBasicAuth
-import shutil
-
-# Extract 'entries', 'description' and 'os_types' fields
-def extract_relevant_fields(filter):
-    return {
-        'entries': filter.get('entries', []),
-        'description': filter.get('description', '')
-    }
-
-# Sort for consistency, so that a hash can be generated
-def sorted_data(value):
-    if isinstance(value, dict):
-        # Recursively sort the dictionary by key
-        return {k: sorted_data(v) for k, v in sorted(value.items())}
-    elif isinstance(value, list):
-        # Sort lists; for dictionaries, sort by a specific key
-        return sorted(value, key=lambda x: tuple(sorted(x.items())) if isinstance(x, dict) else x)
-    return value
-
-# Generate a hash based on sorted relevant fields
-def generate_hash(data):
-    sorted_data_string = json.dumps(sorted_data(data), sort_keys=True)
-    return hashlib.sha256(sorted_data_string.encode('utf-8')).hexdigest()
-
-# Load Elasticsearch credentials from the config file
-def load_credentials(config_path):
-    with open(config_path, 'r') as file:
-        for line in file:
-            if line.startswith("user"):
-                credentials = line.split('=', 1)[1].strip().strip('"')
-                return credentials
-    return None
-
-# Extract username and password from credentials
-def extract_auth_details(credentials):
-    if ':' in credentials:
-        return credentials.split(':', 1)
-    return None, None
-
-# Generalized API request function
-def api_request(method, guid, username, password, json_data=None):
-    headers = {
-        'kbn-xsrf': 'true',
-        'Content-Type': 'application/json'
-    }
-    auth = HTTPBasicAuth(username, password)
-
-    if method == "POST":
-        url = "http://localhost:5601/api/exception_lists/items?namespace_type=agnostic"
-    else:
-        url = f"http://localhost:5601/api/exception_lists/items?item_id={guid}&namespace_type=agnostic"
-
-    response = requests.request(method, url, headers=headers, auth=auth, json=json_data)
-    
-    if response.status_code in [200, 201]:
-        return response.json() if response.content else True
-    elif response.status_code == 404 and method == "GET":
-        return None
-    else:
-        print(f"Error with {method} request: {response.status_code} - {response.text}")
-        return False
-    
-
-# Load YAML data for GUIDs to skip
-def load_disabled(disabled_file_path):
-    if os.path.exists(disabled_file_path):
-        with open(disabled_file_path, 'r') as file:
-            return yaml.safe_load(file) or {}
-    return {}
-
-def load_yaml_files(*dirs):
-    yaml_files = []
-    
-    for dir_path in dirs:
-        if os.path.isdir(dir_path):
-            # Recurse through the directory and subdirectories
-            for root, dirs, files in os.walk(dir_path):
-                for file_name in files:
-                    if file_name.endswith(".yaml"):
-                        full_path = os.path.join(root, file_name)
-                        with open(full_path, 'r') as f:
-                            try:
-                                yaml_content = yaml.safe_load(f)
-                                yaml_files.append(yaml_content)
-                            except yaml.YAMLError as e:
-                                print(f"Error loading {full_path}: {e}")
-        else:
-            print(f"Invalid directory: {dir_path}")
-    
-    return yaml_files
-
-def prepare_custom_rules(input_file, output_dir):
-    # Clear the output directory first
-    if os.path.exists(output_dir):
-        shutil.rmtree(output_dir)
-    os.makedirs(output_dir, exist_ok=True)
-    
-    try:
-        # Load the YAML file
-        with open(input_file, 'r') as f:
-            docs = yaml.safe_load_all(f)
-            
-            for doc in docs:
-                if 'id' not in doc:
-                    print(f"Skipping rule, no 'id' found: {doc}")
-                    continue
-                if doc.get('title') in ["Template 1", "Template 2"]:
-                    print(f"Skipping template rule with title: {doc['title']}")
-                    continue
-                # Create a filename using the 'id' field
-                file_name = os.path.join(output_dir, f"{doc['id']}.yaml")
-                
-                # Write the individual YAML file
-                with open(file_name, 'w') as output_file:
-                    yaml.dump(doc, output_file, default_flow_style=False)
-                print(f"Created file: {file_name}")
-    
-    except yaml.YAMLError as e:
-        print(f"Error parsing YAML: {e}")
-    except Exception as e:
-        print(f"Error processing file: {e}")
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers
@@ -13,16 +13,13 @@

 LOG="/opt/so/log/elasticfleet/so-elastic-agent-gen-installers.log"

-# get the variables needed such as ELASTIC_AGENT_TARBALL_VERSION
-get_elastic_agent_vars
-
 # Check to see if we are already running
 NUM_RUNNING=$(pgrep -cf "/bin/bash /sbin/so-elastic-agent-gen-installers")
 [ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING gen installers script processes running...exiting." >>$LOG && exit 0

 for i in {1..30}
 do
-   ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys?perPage=100" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
+   ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-initial")) | .api_key')
   FLEETHOST=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts/grid-default' | jq -r  '.item.host_urls[]' | paste -sd ',')
 if [[ $FLEETHOST ]] && [[  $ENROLLMENTOKEN ]]; then break; else sleep 10; fi
 done
@@ -39,7 +36,6 @@ printf  "\n### Creating a temp directory at /nsm/elastic-agent-workspace\n"
 rm -rf /nsm/elastic-agent-workspace
 mkdir -p /nsm/elastic-agent-workspace

-
 printf "\n### Extracting outer tarball and then each individual tarball/zip\n"
 tar -xf /nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz -C /nsm/elastic-agent-workspace/
 unzip -q /nsm/elastic-agent-workspace/elastic-agent-*.zip -d /nsm/elastic-agent-workspace/
@@ -76,12 +72,5 @@ do
    printf "\n### $GOOS/$GOARCH Installer Generated...\n"
 done

-printf "\n\n### Generating MSI...\n"
-cp /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64 /opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/so-elastic-agent_windows_amd64.exe
-docker run \
--mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ -w /output \
-{{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} wixl -o so-elastic-agent_windows_amd64_msi --arch x64 /workspace/so-elastic-agent.wxs
-printf "\n### MSI Generated...\n"
-
-printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace\n"
+printf "\n### Cleaning up temp files in /nsm/elastic-agent-workspace"
 rm -rf /nsm/elastic-agent-workspace
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade
@@ -5,7 +5,6 @@
 # this file except in compliance with the Elastic License 2.0.

 . /usr/sbin/so-common
-{%- import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}

 # Only run on Managers
 if ! is_manager_node; then
@@ -28,14 +27,14 @@ OUTDATED_LIST=$(jq -r '.items | map(.id) | (tojson)'  <<<  "$RAW_JSON")

 if [ "$OUTDATED_LIST" != '[]' ]; then
   AGENTNUMBERS=$(jq -r '.total' <<< "$RAW_JSON")
-   printf "Initiating upgrades for $AGENTNUMBERS Agents to Elastic {{ELASTICSEARCHDEFAULTS.elasticsearch.version}}...\n\n"
+   printf "Initiating upgrades for $AGENTNUMBERS Agents to Elastic $ELASTIC_AGENT_TARBALL_VERSION...\n\n"

   # Generate updated JSON payload
-   JSON_STRING=$(jq -n --arg ELASTICVERSION {{ELASTICSEARCHDEFAULTS.elasticsearch.version}} --arg UPDATELIST $OUTDATED_LIST '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')
+   JSON_STRING=$(jq -n --arg ELASTICVERSION $ELASTIC_AGENT_TARBALL_VERSION --arg UPDATELIST $OUTDATED_LIST '{"version": $ELASTICVERSION,"agents": $UPDATELIST }')

   # Update Node Agents
   curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "http://localhost:5601/api/fleet/agents/bulk_upgrade" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 else
    printf "No Agents need updates... Exiting\n\n"
    exit 0
-fi
+fi
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update
@@ -21,104 +21,64 @@ function update_logstash_outputs() {
    # Update Logstash Outputs
    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
 }
-function update_kafka_outputs() {
-  # Make sure SSL configuration is included in policy updates for Kafka output. SSL is configured in so-elastic-fleet-setup
-  SSL_CONFIG=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs/so-manager_kafka" | jq -r '.item.ssl')

-  JSON_STRING=$(jq -n \
-    --arg UPDATEDLIST "$NEW_LIST_JSON" \
-    --argjson SSL_CONFIG "$SSL_CONFIG" \
-    '{"name": "grid-kafka","type": "kafka","hosts": $UPDATEDLIST,"is_default": true,"is_default_monitoring": true,"config_yaml": "","ssl": $SSL_CONFIG}')
-  # Update Kafka outputs
-  curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_kafka" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" | jq
-}
+# Get current list of Logstash Outputs
+RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash')

-{% if GLOBALS.pipeline == "KAFKA" %}
-  # Get current list of Kafka Outputs
-  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_kafka')
+# Check to make sure that the server responded with good data - else, bail from script
+CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
+if [ "$CHECKSUM" != "so-manager_logstash" ]; then
+ printf "Failed to query for current Logstash Outputs..."
+ exit 1
+fi

-  # Check to make sure that the server responded with good data - else, bail from script
-  CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
-  if [ "$CHECKSUM" != "so-manager_kafka" ]; then
-   printf "Failed to query for current Kafka Outputs..."
-   exit 1
-  fi
+# Get the current list of Logstash outputs & hash them
+CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
+CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')

-  # Get the current list of kafka outputs & hash them
-  CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
-  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
-
-  declare -a NEW_LIST=()
-
-  # Query for the current Grid Nodes that are running kafka
-  KAFKANODES=$(salt-call --out=json pillar.get kafka:nodes | jq '.local')
-
-  # Query for Kafka nodes with Broker role and add hostname to list
-  while IFS= read -r line; do
-    NEW_LIST+=("$line")
-  done < <(jq -r 'to_entries | .[] | select(.value.role | contains("broker")) | .key + ":9092"' <<< $KAFKANODES)
-
-  {# If global pipeline isn't set to KAFKA then assume default of REDIS / logstash #}
-{% else %}
-  # Get current list of Logstash Outputs
-  RAW_JSON=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash')
-
-  # Check to make sure that the server responded with good data - else, bail from script
-  CHECKSUM=$(jq -r '.item.id' <<< "$RAW_JSON")
-  if [ "$CHECKSUM" != "so-manager_logstash" ]; then
-   printf "Failed to query for current Logstash Outputs..."
-   exit 1
-  fi
-
-  # Get the current list of Logstash outputs & hash them
-  CURRENT_LIST=$(jq -c -r '.item.hosts' <<<  "$RAW_JSON")
-  CURRENT_HASH=$(sha1sum <<< "$CURRENT_LIST" | awk '{print $1}')
-
-  declare -a NEW_LIST=()
-
-  {# If we select to not send to manager via SOC, then omit the code that adds manager to NEW_LIST #}
-  {% if ELASTICFLEETMERGED.enable_manager_output %}
-  # Create array & add initial elements
-  if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then
-      NEW_LIST+=("{{ GLOBALS.url_base }}:5055")
-  else
-      NEW_LIST+=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055")
-  fi
-  {% endif %}
-
-  # Query for FQDN entries & add them to the list
-  {% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %}
-  CUSTOMFQDNLIST=('{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}')
-  readarray -t -d ' ' CUSTOMFQDN < <(printf '%s' "$CUSTOMFQDNLIST")
-  for CUSTOMNAME in "${CUSTOMFQDN[@]}"
-  do
-   NEW_LIST+=("$CUSTOMNAME:5055")
-  done
-  {% endif %}
-
-  # Query for the current Grid Nodes that are running Logstash
-  LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local')
-
-  # Query for Receiver Nodes & add them to the list
-  if grep -q "receiver" <<< $LOGSTASHNODES; then
-     readarray -t RECEIVERNODES < <(jq -r ' .receiver | keys_unsorted[]'  <<< $LOGSTASHNODES)
-     for NODE in "${RECEIVERNODES[@]}"
-     do
-      NEW_LIST+=("$NODE:5055")
-     done
-  fi
-
-  # Query for Fleet Nodes & add them to the list
-  if grep -q "fleet" <<< $LOGSTASHNODES; then
-     readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]'  <<< $LOGSTASHNODES)
-     for NODE in "${FLEETNODES[@]}"
-     do
-      NEW_LIST+=("$NODE:5055")
-     done
-  fi
+declare -a NEW_LIST=()

+{# If we select to not send to manager via SOC, then omit the code that adds manager to NEW_LIST #}
+{% if ELASTICFLEETMERGED.enable_manager_output %}
+# Create array & add initial elements
+if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then
+    NEW_LIST+=("{{ GLOBALS.url_base }}:5055")
+else
+    NEW_LIST+=("{{ GLOBALS.url_base }}:5055" "{{ GLOBALS.hostname }}:5055")
+fi
 {% endif %}

+# Query for FQDN entries & add them to the list
+{% if ELASTICFLEETMERGED.config.server.custom_fqdn | length > 0 %}
+CUSTOMFQDNLIST=('{{ ELASTICFLEETMERGED.config.server.custom_fqdn | join(' ') }}')
+readarray -t -d ' ' CUSTOMFQDN < <(printf '%s' "$CUSTOMFQDNLIST")
+for CUSTOMNAME in "${CUSTOMFQDN[@]}"
+do
+ NEW_LIST+=("$CUSTOMNAME:5055")
+done
+{% endif %}
+
+# Query for the current Grid Nodes that are running Logstash
+LOGSTASHNODES=$(salt-call --out=json pillar.get logstash:nodes | jq '.local')
+
+# Query for Receiver Nodes & add them to the list
+if grep -q "receiver" <<< $LOGSTASHNODES; then
+   readarray -t RECEIVERNODES < <(jq -r ' .receiver | keys_unsorted[]'  <<< $LOGSTASHNODES)
+   for NODE in "${RECEIVERNODES[@]}"
+   do
+    NEW_LIST+=("$NODE:5055")
+   done
+fi
+
+# Query for Fleet Nodes & add them to the list
+if grep -q "fleet" <<< $LOGSTASHNODES; then
+   readarray -t FLEETNODES < <(jq -r ' .fleet | keys_unsorted[]'  <<< $LOGSTASHNODES)
+   for NODE in "${FLEETNODES[@]}"
+   do
+    NEW_LIST+=("$NODE:5055")
+   done
+fi
+
 # Sort & hash the new list of Logstash Outputs
 NEW_LIST_JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${NEW_LIST[@]}")
 NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
@@ -127,28 +87,9 @@ NEW_HASH=$(sha1sum <<< "$NEW_LIST_JSON" | awk '{print $1}')
 if [ "$NEW_HASH" = "$CURRENT_HASH" ]; then
    printf "\nHashes match - no update needed.\n"
    printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
-
-    # Since output can be KAFKA or LOGSTASH, we need to check if the policy set as default matches the value set in GLOBALS.pipeline and update if needed
-    printf "Checking if the correct output policy is set as default\n"
-    OUTPUT_DEFAULT=$(jq -r '.item.is_default' <<< $RAW_JSON)
-    OUTPUT_DEFAULT_MONITORING=$(jq -r '.item.is_default_monitoring' <<< $RAW_JSON)
-    if [[ "$OUTPUT_DEFAULT" = "false" || "$OUTPUT_DEFAULT_MONITORING" = "false" ]]; then
-      printf "Default output policy needs to be updated.\n"
-    {%- if GLOBALS.pipeline == "KAFKA" and 'gmd' in salt['pillar.get']('features', []) %}
-      update_kafka_outputs
-    {%- else %}
-      update_logstash_outputs
-    {%- endif %}
-    else
-      printf "Default output policy is set - no update needed.\n"
-    fi
    exit 0
 else
    printf "\nHashes don't match - update needed.\n"
    printf "Current List: $CURRENT_LIST\nNew List: $NEW_LIST_JSON\n"
-    {%- if GLOBALS.pipeline == "KAFKA" and 'gmd' in salt['pillar.get']('features', []) %}
-    update_kafka_outputs
-    {%- else %}
    update_logstash_outputs
-    {%- endif %}
 fi
--- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
+++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup
@@ -53,8 +53,7 @@ fi
 printf "\n### Create ES Token ###\n"
 ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' |  jq -r .value)

-### Create Outputs, Fleet Policy and Fleet URLs ###
-# Create the Manager Elasticsearch Output first and set it as the default output
+### Create Outputs & Fleet URLs ###
 printf "\nAdd Manager Elasticsearch Output...\n"
 ESCACRT=$(openssl x509 -in  $INTCA)
 JSON_STRING=$( jq -n \
@@ -63,21 +62,7 @@ JSON_STRING=$( jq -n \
 curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
 printf "\n\n"

-# Create the Manager Fleet Server Host Agent Policy
-# This has to be done while the Elasticsearch Output is set to the default Output
-printf "Create Manager Fleet Server Policy...\n"
-elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "false" "120"
-
-# Modify the default integration policy to update the policy_id with the correct naming
-UPDATED_INTEGRATION_POLICY=$(jq --arg policy_id "FleetServer_{{ GLOBALS.hostname }}" --arg name "fleet_server-{{ GLOBALS.hostname }}" '
-.policy_id = $policy_id |
-.name = $name' /opt/so/conf/elastic-fleet/integrations/fleet-server/fleet-server.json)
-
-# Add the Fleet Server Integration to the new Fleet Policy
-elastic_fleet_integration_create "$UPDATED_INTEGRATION_POLICY"
-
-# Now we can create the Logstash Output and set it to to be the default Output
-printf "\n\nCreate Logstash Output Config if node is not an Import or Eval install\n"
+printf "\nCreate Logstash Output Config if node is not an Import or Eval install\n"
 {% if grains.role not in ['so-import', 'so-eval'] %}
 LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt)
 LOGSTASHKEY=$(openssl rsa -in  /etc/pki/elasticfleet-logstash.key)
@@ -92,11 +77,6 @@ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fl
 printf "\n\n"
 {%- endif %}

-printf "\nCreate Kafka Output Config if node is not an Import or Eval install\n"
-{% if grains.role not in ['so-import', 'so-eval'] %}
-/usr/sbin/so-kafka-fleet-output-policy
-{% endif %}
-
 # Add Manager Hostname & URL Base to Fleet Host URLs
 printf "\nAdd SO-Manager Fleet URL\n"
 if [ "{{ GLOBALS.hostname }}" = "{{ GLOBALS.url_base }}" ]; then
@@ -116,6 +96,16 @@ printf "\n\n"
 # Load Elasticsearch templates
 /usr/sbin/so-elasticsearch-templates-load

+# Manager Fleet Server Host
+elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120"
+
+#Temp Fixup for ES Output bug
+JSON_STRING=$( jq -n \
+                --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \
+                '{"name": $NAME,"description": $NAME,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":120,"data_output_id":"so-manager_elasticsearch"}'
+                )
+curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_{{ GLOBALS.hostname }}" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+
 # Initial Endpoints Policy
 elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600"

@@ -170,4 +160,4 @@ salt-call state.apply elasticfleet queue=True
 # Generate installers & install Elastic Agent on the node
 so-elastic-agent-gen-installers
 salt-call state.apply elasticfleet.install_agent_grid queue=True
-exit 0
+exit 0
--- a/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy
+++ b/salt/elasticfleet/tools/sbin_jinja/so-kafka-fleet-output-policy
@@ -1,52 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% if GLOBALS.role in ['so-manager', 'so-standalone', 'so-managersearch'] %}
-
-. /usr/sbin/so-common
-
-# Check to make sure that Kibana API is up & ready
-RETURN_CODE=0
-wait_for_web_response "http://localhost:5601/api/fleet/settings" "fleet" 300 "curl -K /opt/so/conf/elasticsearch/curl.config"
-RETURN_CODE=$?
-
-if [[ "$RETURN_CODE" != "0" ]]; then
-    printf "Kibana API not accessible, can't setup Elastic Fleet output policy for Kafka..."
-    exit 1
-fi
-
-output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)
-
-if ! echo "$output" | grep -q "so-manager_kafka"; then
-  KAFKACRT=$(openssl x509 -in /etc/pki/elasticfleet-kafka.crt)
-  KAFKAKEY=$(openssl rsa -in  /etc/pki/elasticfleet-kafka.key)
-  KAFKACA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
-  KAFKA_OUTPUT_VERSION="2.6.0"
-  JSON_STRING=$( jq -n \
-                --arg KAFKACRT "$KAFKACRT" \
-                --arg KAFKAKEY "$KAFKAKEY" \
-                --arg KAFKACA "$KAFKACA" \
-                --arg MANAGER_IP "{{ GLOBALS.manager_ip }}:9092" \
-                --arg KAFKA_OUTPUT_VERSION "$KAFKA_OUTPUT_VERSION" \
-                    '{ "name": "grid-kafka", "id": "so-manager_kafka", "type": "kafka", "hosts": [ $MANAGER_IP ], "is_default": false, "is_default_monitoring": false, "config_yaml": "", "ssl": { "certificate_authorities": [ $KAFKACA ], "certificate": $KAFKACRT, "key": $KAFKAKEY, "verification_mode": "full" }, "proxy_id": null, "client_id": "Elastic", "version": $KAFKA_OUTPUT_VERSION, "compression": "none", "auth_type": "ssl", "partition": "round_robin", "round_robin": { "group_events": 1 }, "topics":[{"topic":"%{[event.module]}-securityonion","when":{"type":"regexp","condition":"event.module:.+"}},{"topic":"default-securityonion"}], "headers": [ { "key": "", "value": "" } ], "timeout": 30, "broker_timeout": 30, "required_acks": 1 }'
-                )
-  curl -sK /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" -o /dev/null
-  refresh_output=$(curl -sK /opt/so/conf/elasticsearch/curl.config -L "http://localhost:5601/api/fleet/outputs" | jq -r .items[].id)
-
-  if ! echo "$refresh_output" | grep -q "so-manager_kafka"; then
-    echo -e "\nFailed to setup Elastic Fleet output policy for Kafka...\n"
-    exit 1
-  elif echo "$refresh_output" | grep -q "so-manager_kafka"; then
-    echo -e "\nSuccessfully setup Elastic Fleet output policy for Kafka...\n"
-  fi
-
-elif echo "$output" | grep -q "so-manager_kafka"; then
-  echo -e "\nElastic Fleet output policy for Kafka already exists...\n"
-fi
-{% else %}
-echo -e "\nNo update required...\n"
-{% endif %}
--- a/salt/elasticsearch/config.map.jinja
+++ b/salt/elasticsearch/config.map.jinja
@@ -1,37 +1,23 @@
-{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-   https://securityonion.net/license; you may not use this file except in compliance with the
-   Elastic License 2.0. #}
-
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %}

 {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}

-{# this is a list of dicts containing hostname:ip for elasticsearch nodes that need to know about each other for cluster #}
-{% set ELASTICSEARCH_SEED_HOSTS = [] %}
-{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
+{# ES_LOGSTASH_NODES is the same as LOGSTASH_NODES from logstash/map.jinja but heavynodes and fleet nodes are removed #}
+{% set ES_LOGSTASH_NODES = [] %}
+{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
 {% for node_type, node_details in node_data.items() | sort %}
-{%   if node_type != 'heavynode' %}
+{%   if node_type not in ['heavynode', 'fleet'] %}
 {%     for hostname in node_data[node_type].keys() %}
-{%       do ELASTICSEARCH_SEED_HOSTS.append({hostname:node_details[hostname].ip}) %}
+{%       do ES_LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %}
 {%     endfor %}
 {%   endif %}
 {% endfor %}

-{# this is a list of dicts containing hostname:ip of all nodes running elasticsearch #}
-{% set ELASTICSEARCH_NODES = [] %}
-{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %}
-{% for node_type, node_details in node_data.items() %}
-{%   for hostname in node_data[node_type].keys() %}
-{%     do ELASTICSEARCH_NODES.append({hostname:node_details[hostname].ip}) %}
-{%   endfor %}
-{% endfor %}
-
 {% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %}
-    {% if ELASTICSEARCH_SEED_HOSTS | length > 1 %}
+    {% if ES_LOGSTASH_NODES | length > 1 %}
      {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %}
-      {% for NODE in ELASTICSEARCH_SEED_HOSTS %}
+      {% for NODE in ES_LOGSTASH_NODES %}
        {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %}
      {% endfor %}
    {% endif %}
--- a/salt/elasticsearch/config.sls
+++ b/salt/elasticsearch/config.sls
@@ -118,11 +118,6 @@ esingestconf:
    - user: 930
    - group: 939

-# Remove .fleet_final_pipeline-1 because we are using global@custom now
-so-fleet-final-pipeline-remove:
-  file.absent:
-    - name: /opt/so/conf/elasticsearch/ingest/.fleet_final_pipeline-1
-
 # Auto-generate Elasticsearch ingest node pipelines from pillar
 {% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %}
 es_ingest_conf_{{pipeline}}:
--- a/salt/elasticsearch/defaults.yaml
+++ b/salt/elasticsearch/defaults.yaml
--- a/salt/elasticsearch/download.sls
+++ b/salt/elasticsearch/download.sls
@@ -1,21 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %}
-
-so-elasticsearch_image:
-  docker_image.present:
-    - name: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ ELASTICSEARCHDEFAULTS.elasticsearch.version }}
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}
--- a/salt/elasticsearch/enabled.sls
+++ b/salt/elasticsearch/enabled.sls
@@ -7,8 +7,8 @@
 {% if sls.split('.')[0] in allowed_states %}
 {%   from 'vars/globals.map.jinja' import GLOBALS %}
 {%   from 'docker/docker.map.jinja' import DOCKER %}
-{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %}
-{%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %}
+{%   from 'logstash/map.jinja' import LOGSTASH_NODES %}
+{%   from 'elasticsearch/config.map.jinja' import ES_LOGSTASH_NODES %}
 {%   from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %}
 {%   set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %}
 {%   from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %}
@@ -19,7 +19,7 @@ include:

 so-elasticsearch:
  docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ ELASTICSEARCHMERGED.version }}
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elasticsearch:{{ GLOBALS.so_version }}
    - hostname: elasticsearch
    - name: so-elasticsearch
    - user: elasticsearch
@@ -27,7 +27,7 @@ so-elasticsearch:
      - sobridge:
        - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }}
    - extra_hosts:
-    {% for node in ELASTICSEARCH_NODES %}
+    {% for node in LOGSTASH_NODES %}
    {%   for hostname, ip in node.items() %}
      - {{hostname}}:{{ip}}
    {%   endfor %}
@@ -38,7 +38,7 @@ so-elasticsearch:
      {% endfor %}
    {% endif %}
    - environment:
-      {% if (GLOBALS.role in GLOBALS.manager_roles and ELASTICSEARCH_SEED_HOSTS | length == 1) or GLOBALS.role == 'so-heavynode' %}
+      {% if ES_LOGSTASH_NODES | length == 1 or GLOBALS.role == 'so-heavynode' %}
      - discovery.type=single-node
      {% endif %}
      - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true
@@ -200,15 +200,9 @@ so-elasticsearch-roles-load:
    - require:
      - docker_container: so-elasticsearch
      - file: elasticsearch_sbin_jinja
-
-{%     if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
-{%       if ELASTICSEARCHMERGED.index_clean %}
-{%         set ap = "present" %}
-{%       else %}
-{%         set ap = "absent" %}
-{%       endif %}
+{% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
 so-elasticsearch-indices-delete:
-  cron.{{ap}}:
+  cron.present:
    - name: /usr/sbin/so-elasticsearch-indices-delete > /opt/so/log/elasticsearch/cron-elasticsearch-indices-delete.log 2>&1
    - identifier: so-elasticsearch-indices-delete
    - user: root
@@ -217,8 +211,7 @@ so-elasticsearch-indices-delete:
    - daymonth: '*'
    - month: '*'
    - dayweek: '*'
-{%     endif %}
-
+{% endif %}
 {%   endif %}

 {% else %}
--- a/salt/elasticsearch/files/ingest-dynamic/common
+++ b/salt/elasticsearch/files/ingest-dynamic/common
@@ -62,7 +62,6 @@
    { "split":           { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } },
    { "append":          { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  } },
    { "grok":            { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} },
-    { "set":             { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
    { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
 {%- endraw %}
 {%- if HIGHLANDER %}
@@ -73,9 +72,7 @@
      }
    }
 {%- endif %}
-{%- raw %}
-    ,
-    { "pipeline": { "name": "global@custom", "ignore_missing_pipeline": true, "description": "[Fleet] Global pipeline for all data streams" } }
+{%- raw %} 
  ]
 }
 {% endraw %}
--- a/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1
+++ b/salt/elasticsearch/files/ingest/.fleet_final_pipeline-1
@@ -0,0 +1,106 @@
+{
+  "version": 3,
+  "_meta": {
+    "managed_by": "fleet",
+    "managed": true
+  },
+  "description": "Final pipeline for processing all incoming Fleet Agent documents. \n",
+  "processors": [
+    {
+      "date": {
+        "description": "Add time when event was ingested (and remove sub-seconds to improve storage efficiency)",
+        "tag": "truncate-subseconds-event-ingested",
+        "field": "_ingest.timestamp",
+        "target_field": "event.ingested",
+        "formats": [
+          "ISO8601"
+        ],
+        "output_format": "date_time_no_millis",
+        "ignore_failure": true
+      }
+    },
+    {
+      "remove": {
+        "description": "Remove any pre-existing untrusted values.",
+        "field": [
+          "event.agent_id_status",
+          "_security"
+        ],
+        "ignore_missing": true
+      }
+    },
+    {
+      "set_security_user": {
+        "field": "_security",
+        "properties": [
+          "authentication_type",
+          "username",
+          "realm",
+          "api_key"
+        ]
+      }
+    },
+    {
+      "script": {
+        "description": "Add event.agent_id_status based on the API key metadata and the agent.id contained in the event.\n",
+        "tag": "agent-id-status",
+        "source": "boolean is_user_trusted(def ctx, def users) {\n  if (ctx?._security?.username == null) {\n    return false;\n  }\n\n  def user = null;\n  for (def item : users) {\n    if (item?.username == ctx._security.username) {\n      user = item;\n      break;\n    }\n  }\n\n  if (user == null || user?.realm == null || ctx?._security?.realm?.name == null) {\n    return false;\n  }\n\n  if (ctx._security.realm.name != user.realm) {\n    return false;\n  }\n\n  return true;\n}\n\nString verified(def ctx, def params) {\n  // No agent.id field to validate.\n  if (ctx?.agent?.id == null) {\n    return \"missing\";\n  }\n\n  // Check auth metadata from API key.\n  if (ctx?._security?.authentication_type == null\n      // Agents only use API keys.\n      || ctx._security.authentication_type != 'API_KEY'\n      // Verify the API key owner before trusting any metadata it contains.\n      || !is_user_trusted(ctx, params.trusted_users)\n      // Verify the API key has metadata indicating the assigned agent ID.\n      || ctx?._security?.api_key?.metadata?.agent_id == null) {\n    return \"auth_metadata_missing\";\n  }\n\n  // The API key can only be used represent the agent.id it was issued to.\n  if (ctx._security.api_key.metadata.agent_id != ctx.agent.id) {\n    // Potential masquerade attempt.\n    return \"mismatch\";\n  }\n\n  return \"verified\";\n}\n\nif (ctx?.event == null) {\n  ctx.event = [:];\n}\n\nctx.event.agent_id_status = verified(ctx, params);",
+        "params": {
+          "trusted_users": [
+            {
+              "username": "elastic/fleet-server",
+              "realm": "_service_account"
+            },
+            {
+              "username": "cloud-internal-agent-server",
+              "realm": "found"
+            },
+            {
+              "username": "elastic",
+              "realm": "reserved"
+            }
+          ]
+        }
+      }
+    },
+    {
+      "remove": {
+        "field": "_security",
+        "ignore_missing": true
+      }
+    }, 
+    { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
+    { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
+    { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
+    { "gsub":       { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
+    { "append":     { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}"  } },
+    { "set":        { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
+    { "set":        { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
+    { "set":        { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
+    { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
+    { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.dataset", "value": "import" } },
+    { "set":        { "if": "ctx.tags.0 == 'import'", "override": true, "field": "data_stream.namespace", "value": "so" } },
+    { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp", "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
+    { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
+    { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
+    { "rename":        { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
+    { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } 
+  ],
+  "on_failure": [
+    {
+      "remove": {
+        "field": "_security",
+        "ignore_missing": true,
+        "ignore_failure": true
+      }
+    },
+    {
+      "append": {
+        "field": "error.message",
+        "value": [
+          "failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}"
+        ]
+      }
+    }
+  ]
+}
--- a/salt/elasticsearch/files/ingest/global@custom
+++ b/salt/elasticsearch/files/ingest/global@custom
@@ -1,27 +0,0 @@
-{
-  "version": 3,
-  "_meta": {
-    "managed_by": "securityonion",
-    "managed": true
-  },
-  "description": "Custom pipeline for processing all incoming Fleet Agent documents. \n",
-  "processors": [
-    { "set":        { "ignore_failure": true,  "field": "event.module", "value": "elastic_agent" } },
-    { "split":      { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "module_temp"  } },
-    { "set":        { "if": "ctx.module_temp != null", "override": true, "field": "event.module", "value": "{{module_temp.0}}" } },
-    { "gsub":       { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "pattern": "^[^.]*.", "replacement": "", "target_field": "dataset_tag_temp" } },
-    { "append":     { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp}}", "allow_duplicates": false  } },
-    { "set":        { "if": "ctx.network?.direction == 'egress'", "override": true, "field": "network.initiated", "value": "true" } },
-    { "set":        { "if": "ctx.network?.direction == 'ingress'", "override": true, "field": "network.initiated", "value": "false" } },
-    { "set":        { "if": "ctx.network?.type == 'ipv4'", "override": true, "field": "destination.ipv6", "value": "false" } },
-    { "set":        { "if": "ctx.network?.type == 'ipv6'", "override": true, "field": "destination.ipv6", "value": "true" } },
-    { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.dataset", "value": "import" } },
-    { "set":        { "if": "ctx.tags != null && ctx.tags.contains('import')", "override": true, "field": "data_stream.namespace", "value": "so" } },
-    { "date":       { "if": "ctx.event?.module == 'system'", "field": "event.created", "target_field": "@timestamp","ignore_failure": true, "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSX","yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'"] } },
-    { "community_id":{ "if": "ctx.event?.dataset == 'endpoint.events.network'", "ignore_failure":true }  },
-    { "set":        { "if": "ctx.event?.module == 'fim'", "override": true, "field": "event.module", "value": "file_integrity" } },
-    { "rename":        { "if": "ctx.winlog?.provider_name == 'Microsoft-Windows-Windows Defender'", "ignore_missing": true, "field": "winlog.event_data.Threat Name", "target_field": "winlog.event_data.threat_name" } },
-    { "set":        { "if": "ctx?.metadata?.kafka != null" , "field": "kafka.id", "value": "{{metadata.kafka.partition}}{{metadata.kafka.offset}}{{metadata.kafka.timestamp}}", "ignore_failure": true } },
-    { "remove":     { "field": [ "message2", "type", "fields", "category", "module", "dataset", "event.dataset_temp", "dataset_tag_temp", "module_temp" ], "ignore_missing": true, "ignore_failure": true } } 
-  ]
-}
--- a/salt/elasticsearch/files/ingest/hydra
+++ b/salt/elasticsearch/files/ingest/hydra
@@ -1,9 +0,0 @@
-{
-  "description" : "hydra",
-  "processors" : [
-    {"set":{"field":"audience","value":"access","override":false,"ignore_failure":true}},
-    {"set":{"field":"event.dataset","ignore_empty_value":true,"ignore_failure":true,"value":"hydra.{{{audience}}}","media_type":"text/plain"}},
-    {"set":{"field":"event.action","ignore_failure":true,"copy_from":"msg"    }},
-    { "pipeline":    { "name": "common" } }
-  ]
-}
--- a/salt/elasticsearch/files/ingest/kismet.ad_hoc
+++ b/salt/elasticsearch/files/ingest/kismet.ad_hoc
@@ -1,10 +0,0 @@
-{
-  "processors": [
-    {
-      "rename": {
-        "field": "message2.kismet_device_base_macaddr",
-        "target_field": "network.wireless.bssid"
-      }
-    }
-  ]
-}
--- a/salt/elasticsearch/files/ingest/kismet.ap
+++ b/salt/elasticsearch/files/ingest/kismet.ap
@@ -1,50 +0,0 @@
-{
-  "processors": [
-    {
-      "rename": {
-        "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_cloaked",
-        "target_field": "network.wireless.ssid_cloaked",
-        "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_cloaked != null"
-      }
-    },
-    {
-      "rename": {
-        "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_ssid",
-        "target_field": "network.wireless.ssid",
-        "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_ssid != null"
-      }
-    },
-    {
-      "set": {
-        "field": "network.wireless.ssid",
-        "value": "Hidden",
-        "if": "ctx?.network?.wireless?.ssid_cloaked != null && ctx?.network?.wireless?.ssid_cloaked == 1"
-      }
-    },
-    {
-      "rename": {
-        "field": "message2.dot11_device.dot11_device_last_beaconed_ssid_record.dot11_advertisedssid_dot11e_channel_utilization_perc",
-        "target_field": "network.wireless.channel_utilization",
-        "if": "ctx?.message2?.dot11_device?.dot11_device_last_beaconed_ssid_record?.dot11_advertisedssid_dot11e_channel_utilization_perc != null"
-      }
-    },
-    {
-      "rename": {
-        "field": "message2.dot11_device.dot11_device_last_bssid",
-        "target_field": "network.wireless.bssid"
-      }
-    },
-    {
-      "foreach": {
-        "field": "message2.dot11_device.dot11_device_associated_client_map",
-        "processor": {
-          "append": {
-            "field": "network.wireless.associated_clients",
-            "value": "{{_ingest._key}}"
-          }
-        },
-        "if": "ctx?.message2?.dot11_device?.dot11_device_associated_client_map != null"
-      }
-    }
-  ]
-}
--- a/salt/elasticsearch/files/ingest/kismet.bridged
+++ b/salt/elasticsearch/files/ingest/kismet.bridged
@@ -1,16 +0,0 @@
-{
-  "processors": [
-    {
-      "rename": {
-        "field": "message2.kismet_device_base_macaddr",
-        "target_field": "client.mac"
-      }
-    },
-    {
-      "rename": {
-        "field": "message2.dot11_device.dot11_device_last_bssid",
-        "target_field": "network.wireless.bssid"
-      }
-    }
-  ]
-}
--- a/salt/elasticsearch/files/ingest/kismet.client
+++ b/salt/elasticsearch/files/ingest/kismet.client
@@ -1,29 +0,0 @@
-{
-  "processors": [
-    {
-      "rename": {
-        "field": "message2.kismet_device_base_macaddr",
-        "target_field": "client.mac"
-      }
-    },
-    {
-      "rename": {
-        "field": "message2.dot11_device.dot11_device_last_bssid",
-        "target_field": "network.wireless.last_connected_bssid",
-        "if": "ctx?.message2?.dot11_device?.dot11_device_last_bssid != null"
-      }
-    },
-    {
-      "foreach": {
-        "field": "message2.dot11_device.dot11_device_client_map",
-        "processor": {
-          "append": {
-            "field": "network.wireless.known_connected_bssid",
-            "value": "{{_ingest._key}}"
-          }
-        },
-        "if": "ctx?.message2?.dot11_device?.dot11_device_client_map != null"
-      }
-    }
-  ]
-}
--- a/salt/elasticsearch/files/ingest/kismet.common
+++ b/salt/elasticsearch/files/ingest/kismet.common
@@ -1,159 +0,0 @@
-{
-  "processors": [
-    {
-      "json": {
-        "field": "message",
-        "target_field": "message2"
-      }
-    },
-    {
-      "date": {
-        "field": "message2.kismet_device_base_mod_time",
-        "formats": [
-          "epoch_second"
-        ],
-        "target_field": "@timestamp"
-      }
-    },
-    {
-      "set": {
-        "field": "event.category",
-        "value": "network"
-      }
-    },
-    {
-      "dissect": {
-        "field": "message2.kismet_device_base_type",
-        "pattern": "%{wifi} %{device_type}"
-      }
-    },
-    {
-      "lowercase": {
-        "field": "device_type"
-      }
-    },
-    {
-      "set": {
-        "field": "event.dataset",
-        "value": "kismet.{{device_type}}"
-      }
-    },
-    {
-      "set": {
-        "field": "event.dataset",
-        "value": "kismet.wds_ap",
-        "if": "ctx?.device_type == 'wds ap'"
-      }
-    },
-    {
-      "set": {
-        "field": "event.dataset",
-        "value": "kismet.ad_hoc",
-        "if": "ctx?.device_type == 'ad-hoc'"
-      }
-    },
-    {
-      "set": {
-        "field": "event.module",
-        "value": "kismet"
-      }
-    },
-    {
-      "rename": {
-        "field": "message2.kismet_device_base_packets_tx_total",
-        "target_field": "source.packets"
-      }
-    },
-    {
-      "rename": {
-        "field": "message2.kismet_device_base_num_alerts",
-        "target_field": "kismet.alerts.count"
-      }
-    },
-    {
-      "rename": {
-        "field": "message2.kismet_device_base_channel",
-        "target_field": "network.wireless.channel",
-        "if": "ctx?.message2?.kismet_device_base_channel != ''"
-      }
-    },
-    {
-      "rename": {
-        "field": "message2.kismet_device_base_frequency",
-        "target_field": "network.wireless.frequency",
-        "if": "ctx?.message2?.kismet_device_base_frequency != 0"
-      }
-    },
-    {
-      "rename": {
-        "field": "message2.kismet_device_base_last_time",
-        "target_field": "kismet.last_seen"
-      }
-    },
-    {
-      "date": {
-        "field": "kismet.last_seen",
-        "formats": [
-          "epoch_second"
-        ],
-        "target_field": "kismet.last_seen"
-      }
-    },
-    {
-      "rename": {
-        "field": "message2.kismet_device_base_first_time",
-        "target_field": "kismet.first_seen"
-      }
-    },
-    {
-      "date": {
-        "field": "kismet.first_seen",
-        "formats": [
-          "epoch_second"
-        ],
-        "target_field": "kismet.first_seen"
-      }
-    },
-    {
-      "rename": {
-        "field": "message2.kismet_device_base_seenby",
-        "target_field": "kismet.seenby"
-      }
-    },
-    {
-      "foreach": {
-        "field": "kismet.seenby",
-        "processor": {
-          "pipeline": {
-            "name": "kismet.seenby"
-          }
-        }
-      }
-    },
-    {
-      "rename": {
-        "field": "message2.kismet_device_base_manuf",
-        "target_field": "device.manufacturer"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "{{event.dataset}}"
-      }
-    },
-    {
-      "remove": {
-        "field": [
-          "message2",
-          "message",
-          "device_type",
-          "wifi",
-          "agent",
-          "host",
-          "event.created"
-        ],
-        "ignore_failure": true
-      }
-    }
-  ]
-}
--- a/salt/elasticsearch/files/ingest/kismet.device
+++ b/salt/elasticsearch/files/ingest/kismet.device
@@ -1,9 +0,0 @@
-{
-  "processors": [
-    {
-      "pipeline": {
-        "name": "kismet.client"
-      }
-    }
-  ]
-}
--- a/salt/elasticsearch/files/ingest/kismet.seenby
+++ b/salt/elasticsearch/files/ingest/kismet.seenby
@@ -1,52 +0,0 @@
-{
-  "processors": [
-    {
-      "rename": {
-        "field": "_ingest._value.kismet_common_seenby_num_packets",
-        "target_field": "_ingest._value.packets_seen",
-        "ignore_missing": true
-      }
-    },
-    {
-      "rename": {
-        "field": "_ingest._value.kismet_common_seenby_uuid",
-        "target_field": "_ingest._value.serial_number",
-        "ignore_missing": true
-      }
-    },
-    {
-      "rename": {
-        "field": "_ingest._value.kismet_common_seenby_first_time",
-        "target_field": "_ingest._value.first_seen",
-        "ignore_missing": true
-      }
-    },
-    {
-      "rename": {
-        "field": "_ingest._value.kismet_common_seenby_last_time",
-        "target_field": "_ingest._value.last_seen",
-        "ignore_missing": true
-      }
-    },
-    {
-      "date": {
-        "field": "_ingest._value.first_seen",
-        "formats": [
-          "epoch_second"
-        ],
-        "target_field": "_ingest._value.first_seen",
-        "ignore_failure": true
-      }
-    },
-    {
-      "date": {
-        "field": "_ingest._value.last_seen",
-        "formats": [
-          "epoch_second"
-        ],
-        "target_field": "_ingest._value.last_seen",
-        "ignore_failure": true
-      }
-    }
-  ]
-}
--- a/salt/elasticsearch/files/ingest/kismet.wds
+++ b/salt/elasticsearch/files/ingest/kismet.wds
@@ -1,10 +0,0 @@
-{
-  "processors": [
-    {
-      "rename": {
-        "field": "message2.kismet_device_base_macaddr",
-        "target_field": "client.mac"
-      }
-    }
-  ]
-}
--- a/salt/elasticsearch/files/ingest/kismet.wds_ap
+++ b/salt/elasticsearch/files/ingest/kismet.wds_ap
@@ -1,22 +0,0 @@
-{
-  "processors": [
-    {
-      "rename": {
-        "field": "message2.kismet_device_base_commonname",
-        "target_field": "network.wireless.bssid"
-      }
-    },
-    {
-      "foreach": {
-        "field": "message2.dot11_device.dot11_device_associated_client_map",
-        "processor": {
-          "append": {
-            "field": "network.wireless.associated_clients",
-            "value": "{{_ingest._key}}"
-          }
-        },
-        "if": "ctx?.message2?.dot11_device?.dot11_device_associated_client_map != null"
-      }
-    }
-  ]
-}
--- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0
+++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0
@@ -1,5 +1,5 @@
 {
-  "description": "Pipeline for PFsense",
+  "description": "Pipeline for pfSense",
  "processors": [
    {
      "set": {
--- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0-suricata
+++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0-suricata
@@ -1,14 +1,9 @@
 {
  "description": "Pipeline for parsing pfSense Suricata logs.",
  "processors": [
-    { "set": {
-        "field": "event.module",
-        "value": "suricata"
-      }
-    },
    {
     "pipeline": {
-        "name": "suricata.common_pfsense"
+        "name": "suricata.common"
      }
    }
  ],
--- a/salt/elasticsearch/files/ingest/logs-pfsense.log-1.19.1
+++ b/salt/elasticsearch/files/ingest/logs-pfsense.log-1.19.1
@@ -1,414 +0,0 @@
-{
-  "description": "Pipeline for PFsense",
-  "processors": [
-    {
-      "set": {
-        "field": "ecs.version",
-        "value": "8.11.0"
-      }
-    },
-    {
-      "set": {
-        "field": "observer.vendor",
-        "value": "netgate"
-      }
-    },
-    {
-      "set": {
-        "field": "observer.type",
-        "value": "firewall"
-      }
-    },
-    {
-      "rename": {
-        "field": "message",
-        "target_field": "event.original",
-        "ignore_missing": true,
-        "if": "ctx.event?.original == null"
-      }
-    },
-    {
-      "set": {
-        "field": "event.kind",
-        "value": "event"
-      }
-    },
-    {
-      "set": {
-        "field": "event.timezone",
-        "value": "{{_tmp.tz_offset}}",
-        "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'"
-      }
-    },
-    {
-      "grok": {
-        "description": "Parse syslog header",
-        "field": "event.original",
-        "patterns": [
-          "^(%{ECS_SYSLOG_PRI})?%{TIMESTAMP} %{GREEDYDATA:message}"
-        ],
-        "pattern_definitions": {
-          "ECS_SYSLOG_PRI": "<%{NONNEGINT:log.syslog.priority:long}>(\\d )?",
-          "TIMESTAMP": "(?:%{BSD_TIMESTAMP_FORMAT}|%{SYSLOG_TIMESTAMP_FORMAT})",
-          "BSD_TIMESTAMP_FORMAT": "%{SYSLOGTIMESTAMP:_tmp.timestamp}(%{SPACE}%{BSD_PROCNAME}|%{SPACE}%{OBSERVER}%{SPACE}%{BSD_PROCNAME})(\\[%{POSINT:process.pid:long}\\])?:",
-          "BSD_PROCNAME": "(?:\\b%{NAME:process.name}|\\(%{NAME:process.name}\\))",
-          "NAME": "[[[:alnum:]]_-]+",
-          "SYSLOG_TIMESTAMP_FORMAT": "%{TIMESTAMP_ISO8601:_tmp.timestamp8601}%{SPACE}%{OBSERVER}%{SPACE}%{PROCESS}%{SPACE}(%{POSINT:process.pid:long}|-) - (-|%{META})",
-          "TIMESTAMP_ISO8601": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE:event.timezone}?",
-          "OBSERVER": "(?:%{IP:observer.ip}|%{HOSTNAME:observer.name})",
-          "UNIXPATH": "(/([\\w_%!$@:.,+~-]+|\\\\.)*)*",
-          "PROCESS": "(\\(%{DATA:process.name}\\)|(?:%{UNIXPATH})%{BASEPATH:process.name})",
-          "BASEPATH": "[[[:alnum:]]_%!$@:.,+~-]+",
-          "META": "\\[[^\\]]*\\]"
-        }
-      }
-    },
-    {
-      "date": {
-        "if": "ctx._tmp.timestamp8601 != null",
-        "field": "_tmp.timestamp8601",
-        "target_field": "@timestamp",
-        "formats": [
-          "ISO8601"
-        ]
-      }
-    },
-    {
-      "date": {
-        "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null",
-        "field": "_tmp.timestamp",
-        "target_field": "@timestamp",
-        "formats": [
-          "MMM  d HH:mm:ss",
-          "MMM d HH:mm:ss",
-          "MMM dd HH:mm:ss"
-        ],
-        "timezone": "{{ event.timezone }}"
-      }
-    },
-    {
-      "grok": {
-        "description": "Set Event Provider",
-        "field": "process.name",
-        "patterns": [
-          "^%{HYPHENATED_WORDS:event.provider}"
-        ],
-        "pattern_definitions": {
-          "HYPHENATED_WORDS": "\\b[A-Za-z0-9_]+(-[A-Za-z_]+)*\\b"
-        }
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-firewall",
-        "if": "ctx.event.provider == 'filterlog'"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-openvpn",
-        "if": "ctx.event.provider == 'openvpn'"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-ipsec",
-        "if": "ctx.event.provider == 'charon'"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-dhcp",
-        "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-unbound",
-        "if": "ctx.event.provider == 'unbound'"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-haproxy",
-        "if": "ctx.event.provider == 'haproxy'"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-php-fpm",
-        "if": "ctx.event.provider == 'php-fpm'"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log-1.19.1-squid",
-        "if": "ctx.event.provider == 'squid'"
-      }
-    },
-    {
-     "pipeline": {
-        "name": "logs-pfsense.log-1.16.0-suricata",
-        "if": "ctx.event.provider == 'suricata'"
-      }
-    },
-    {
-      "drop": {
-        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"suricata\"].contains(ctx.event?.provider)"
-      }
-    },
-    {
-      "append": {
-        "field": "event.category",
-        "value": "network",
-        "if": "ctx.network != null"
-      }
-    },
-    {
-      "convert": {
-        "field": "source.address",
-        "target_field": "source.ip",
-        "type": "ip",
-        "ignore_failure": true,
-        "ignore_missing": true
-      }
-    },
-    {
-      "convert": {
-        "field": "destination.address",
-        "target_field": "destination.ip",
-        "type": "ip",
-        "ignore_failure": true,
-        "ignore_missing": true
-      }
-    },
-    {
-      "set": {
-        "field": "network.type",
-        "value": "ipv6",
-        "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")"
-      }
-    },
-    {
-      "set": {
-        "field": "network.type",
-        "value": "ipv4",
-        "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")"
-      }
-    },
-    {
-      "geoip": {
-        "field": "source.ip",
-        "target_field": "source.geo",
-        "ignore_missing": true
-      }
-    },
-    {
-      "geoip": {
-        "field": "destination.ip",
-        "target_field": "destination.geo",
-        "ignore_missing": true
-      }
-    },
-    {
-      "geoip": {
-        "ignore_missing": true,
-        "database_file": "GeoLite2-ASN.mmdb",
-        "field": "source.ip",
-        "target_field": "source.as",
-        "properties": [
-          "asn",
-          "organization_name"
-        ]
-      }
-    },
-    {
-      "geoip": {
-        "database_file": "GeoLite2-ASN.mmdb",
-        "field": "destination.ip",
-        "target_field": "destination.as",
-        "properties": [
-          "asn",
-          "organization_name"
-        ],
-        "ignore_missing": true
-      }
-    },
-    {
-      "rename": {
-        "field": "source.as.asn",
-        "target_field": "source.as.number",
-        "ignore_missing": true
-      }
-    },
-    {
-      "rename": {
-        "field": "source.as.organization_name",
-        "target_field": "source.as.organization.name",
-        "ignore_missing": true
-      }
-    },
-    {
-      "rename": {
-        "field": "destination.as.asn",
-        "target_field": "destination.as.number",
-        "ignore_missing": true
-      }
-    },
-    {
-      "rename": {
-        "field": "destination.as.organization_name",
-        "target_field": "destination.as.organization.name",
-        "ignore_missing": true
-      }
-    },
-    {
-      "community_id": {
-        "target_field": "network.community_id",
-        "ignore_failure": true
-      }
-    },
-    {
-      "grok": {
-        "field": "observer.ingress.interface.name",
-        "patterns": [
-          "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}"
-        ],
-        "ignore_missing": true,
-        "ignore_failure": true
-      }
-    },
-    {
-      "set": {
-        "field": "network.vlan.id",
-        "copy_from": "observer.ingress.vlan.id",
-        "ignore_empty_value": true
-      }
-    },
-    {
-      "append": {
-        "field": "related.ip",
-        "value": "{{destination.ip}}",
-        "allow_duplicates": false,
-        "if": "ctx.destination?.ip != null"
-      }
-    },
-    {
-      "append": {
-        "field": "related.ip",
-        "value": "{{source.ip}}",
-        "allow_duplicates": false,
-        "if": "ctx.source?.ip != null"
-      }
-    },
-    {
-      "append": {
-        "field": "related.ip",
-        "value": "{{source.nat.ip}}",
-        "allow_duplicates": false,
-        "if": "ctx.source?.nat?.ip != null"
-      }
-    },
-    {
-      "append": {
-        "field": "related.hosts",
-        "value": "{{destination.domain}}",
-        "if": "ctx.destination?.domain != null"
-      }
-    },
-    {
-      "append": {
-        "field": "related.user",
-        "value": "{{user.name}}",
-        "if": "ctx.user?.name != null"
-      }
-    },
-    {
-      "set": {
-        "field": "network.direction",
-        "value": "{{network.direction}}bound",
-        "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/"
-      }
-    },
-    {
-      "remove": {
-        "field": [
-          "_tmp"
-        ],
-        "ignore_failure": true
-      }
-    },
-    {
-      "script": {
-        "lang": "painless",
-        "description": "This script processor iterates over the whole document to remove fields with null values.",
-        "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
-      }
-    },
-    {
-      "remove": {
-        "field": "event.original",
-        "if": "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))",
-        "ignore_failure": true,
-        "ignore_missing": true
-      }
-    },
-    {
-      "pipeline": {
-        "name": "global@custom",
-        "ignore_missing_pipeline": true,
-        "description": "[Fleet] Global pipeline for all data streams"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs@custom",
-        "ignore_missing_pipeline": true,
-        "description": "[Fleet] Pipeline for all data streams of type `logs`"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.integration@custom",
-        "ignore_missing_pipeline": true,
-        "description": "[Fleet] Pipeline for all data streams of type `logs` defined by the `pfsense` integration"
-      }
-    },
-    {
-      "pipeline": {
-        "name": "logs-pfsense.log@custom",
-        "ignore_missing_pipeline": true,
-        "description": "[Fleet] Pipeline for the `pfsense.log` dataset"
-      }
-    }
-  ],
-  "on_failure": [
-    {
-      "remove": {
-        "field": [
-          "_tmp"
-        ],
-        "ignore_failure": true
-      }
-    },
-    {
-      "set": {
-        "field": "event.kind",
-        "value": "pipeline_error"
-      }
-    },
-    {
-      "append": {
-        "field": "error.message",
-        "value": "{{{ _ingest.on_failure_message }}}"
-      }
-    }
-  ],
-  "_meta": {
-    "managed_by": "fleet",
-    "managed": true,
-    "package": {
-      "name": "pfsense"
-    }
-  }
-}
--- a/salt/elasticsearch/files/ingest/strelka.file
+++ b/salt/elasticsearch/files/ingest/strelka.file
@@ -56,7 +56,6 @@
    { "set":         { "if": "ctx.exiftool?.Subsystem != null", "field": "host.subsystem",        "value": "{{exiftool.Subsystem}}", "ignore_failure": true }},
    { "set":         { "if": "ctx.scan?.yara?.matches instanceof List", "field": "rule.name",        "value": "{{scan.yara.matches.0}}" }},
    { "set":         { "if": "ctx.rule?.name != null", "field": "event.dataset",        "value": "alert", "override": true }},
-    { "set":         { "if": "ctx.rule?.name != null", "field": "rule.uuid",        "value": "{{rule.name}}", "override": true }},
    { "rename":         { "field": "file.flavors.mime", "target_field": "file.mime_type",        "ignore_missing": true }},
    { "set":         { "if": "ctx.rule?.name != null && ctx.rule?.score == null",   "field": "event.severity", "value": 3, "override": true }  },
    { "convert" :    { "if": "ctx.rule?.score != null", "field" : "rule.score","type": "integer"}},
--- a/salt/elasticsearch/files/ingest/suricata.alert
+++ b/salt/elasticsearch/files/ingest/suricata.alert
@@ -1,7 +1,6 @@
 {
  "description" : "suricata.alert",
  "processors" : [
-    { "set":   { "if": "ctx.event?.imported != true", "field": "_index", "value": "logs-suricata.alerts-so" } },
    { "set":   { "field": "tags","value": "alert" }},
    { "rename":{ "field": "message2.alert",	"target_field": "rule",	"ignore_failure": true 	} },
    { "rename":{ "field": "rule.signature",     "target_field": "rule.name", "ignore_failure": true  } },
--- a/salt/elasticsearch/files/ingest/suricata.alert_pfsense
+++ b/salt/elasticsearch/files/ingest/suricata.alert_pfsense
@@ -1,16 +0,0 @@
-{
-  "description" : "suricata.alert",
-  "processors" : [
-    { "set":   { "field": "data_stream.dataset", "value": "suricata" } },
-    { "set":   { "field": "data_stream.namespace", "value": "so" } },
-    { "set":   { "field": "_index", "value": "logs-suricata.alerts-so" } },
-    { "set":   { "field": "tags","value": "alert" }},
-    { "rename":{ "field": "message2.alert",	"target_field": "rule",	"ignore_failure": true 	} },
-    { "rename":{ "field": "rule.signature",     "target_field": "rule.name", "ignore_failure": true  } },
-    { "rename":{ "field": "rule.ref",     "target_field": "rule.version", "ignore_failure": true  } },
-    { "rename":{ "field": "rule.signature_id",     "target_field": "rule.uuid", "ignore_failure": true  } },
-    { "rename":{ "field": "rule.signature_id",     "target_field": "rule.signature", "ignore_failure": true  } },
-    { "rename":{ "field": "message2.payload_printable",     "target_field": "network.data.decoded", "ignore_failure": true  } },
-    { "pipeline": { "name": "common.nids" } }
-  ]
-}
--- a/salt/elasticsearch/files/ingest/suricata.common_pfsense
+++ b/salt/elasticsearch/files/ingest/suricata.common_pfsense
@@ -1,23 +0,0 @@
-{
-  "description" : "suricata.common",
-  "processors" : [
-    { "json":   { "field": "message",                   "target_field": "message2",             "ignore_failure": true  } },
-    { "rename": { "field": "message2.pkt_src",          "target_field": "network.packet_source","ignore_failure": true  } },
-    { "rename": { "field": "message2.proto",            "target_field": "network.transport",    "ignore_failure": true  } },
-    { "rename": { "field": "message2.in_iface",         "target_field": "observer.ingress.interface.name",    "ignore_failure": true  } },
-    { "rename": { "field": "message2.flow_id",          "target_field": "log.id.uid",           "ignore_failure": true  } },
-    { "rename": { "field": "message2.src_ip",           "target_field": "source.ip",            "ignore_failure": true  } },
-    { "rename": { "field": "message2.src_port",         "target_field": "source.port",          "ignore_failure": true  } },
-    { "rename": { "field": "message2.dest_ip",          "target_field": "destination.ip",       "ignore_failure": true  } },
-    { "rename": { "field": "message2.dest_port",        "target_field": "destination.port",     "ignore_failure": true  } },
-    { "rename": { "field": "message2.vlan",             "target_field": "network.vlan.id",      "ignore_failure": true  } },
-    { "rename": { "field": "message2.community_id",     "target_field": "network.community_id", "ignore_missing": true  } },
-    { "rename": { "field": "message2.xff",              "target_field": "xff.ip",               "ignore_missing": true  } },
-    { "set":    { "field": "event.dataset",             "value": "{{ message2.event_type }}"                            } },
-    { "set":    { "field": "observer.name",             "value": "{{agent.name}}"                                       } },
-    { "set":    { "field": "event.ingested",            "value": "{{@timestamp}}"                                       } },
-    { "date": { "field": "message2.timestamp", "target_field": "@timestamp", "formats": ["ISO8601", "UNIX"], "timezone": "UTC", "ignore_failure": true } },
-    { "remove":{ "field": "agent",                                                              "ignore_failure": true  } },
-    { "pipeline": { "if": "ctx?.event?.dataset != null", "name": "suricata.{{event.dataset}}_pfsense" } }
-  ]
-}
--- a/salt/elasticsearch/files/ingest/zeek.common
+++ b/salt/elasticsearch/files/ingest/zeek.common
@@ -18,7 +18,6 @@
    { "set":         { "if": "ctx.destination?.ip != null", "field": "server.ip",        "value": "{{destination.ip}}"  } },
    { "set":         { "if": "ctx.destination?.port != null", "field": "server.port",        "value": "{{destination.port}}"  } },
    { "set":         { "field": "observer.name",        "value": "{{agent.name}}"  } },
-    { "append": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"openvpn\")","field": "tags","value": ["{{network.protocol}}"],"allow_duplicates": false,"ignore_failure": true}},
    { "date":		{ "field": "message2.ts",	"target_field": "@timestamp",	"formats": ["ISO8601", "UNIX"], "ignore_failure": true	} },
    { "remove":		{ "field": ["agent"],	"ignore_failure": true									} },
    { "pipeline":	{ "name": "common" 													} }
--- a/salt/elasticsearch/files/ingest/zeek.conn
+++ b/salt/elasticsearch/files/ingest/zeek.conn
@@ -38,8 +38,6 @@
    { "set": { "if": "ctx.connection?.state == 'SH'",    "field": "connection.state_description", "value": "Originator sent a SYN followed by a FIN, we never saw a SYN ACK from the responder (hence the connection was 'half' open)" } },
    { "set": { "if": "ctx.connection?.state == 'SHR'",   "field": "connection.state_description", "value": "Responder sent a SYN ACK followed by a FIN, we never saw a SYN from the originator" } },
    { "set": { "if": "ctx.connection?.state == 'OTH'",   "field": "connection.state_description", "value": "No SYN seen, just midstream traffic (a 'partial connection' that was not later closed)" } },
-    { "set": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"ipsec\")", "field": "network.protocol", "value": "ipsec"}},
-    { "set": { "if": "ctx.network?.protocol != null && ctx.network?.protocol.contains(\"openvpn\")", "field": "network.protocol", "value": "openvpn"}},
    { "pipeline": 	{ "name": "zeek.common" } }
  ]
 }
--- a/salt/elasticsearch/files/ingest/zeek.http2
+++ b/salt/elasticsearch/files/ingest/zeek.http2
@@ -1,37 +0,0 @@
-{
-  "description" : "zeek.http2",
-  "processors" : [
-    { "set":      { "field": "event.dataset",                   "value": "http2" } },
-    { "set":      { "field": "network.transport",               "value": "tcp"  } },
-    { "json":     { "field": "message",                         "target_field": "message2",                       "ignore_failure": true  } },
-    { "rename": 	{ "field": "message2.trans_depth",	          "target_field": "http.trans_depth",		        "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.method", 		            "target_field": "http.method",		            "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.host", 		              "target_field": "http.virtual_host",		      "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.uri", 		                "target_field": "http.uri",			              "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.referrer", 	            "target_field": "http.referrer",		          "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.version", 		            "target_field": "http.version",		            "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.user_agent", 	          "target_field": "http.useragent",		          "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.request_body_len",       "target_field": "http.request.body.length",	  "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.response_body_len",      "target_field": "http.response.body.length",	"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.status_code",	          "target_field": "http.status_code",		        "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.status_msg", 	          "target_field": "http.status_message",	      "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.info_code", 	            "target_field": "http.info_code",		          "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.info_msg", 	            "target_field": "http.info_message",		      "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.username", 	            "target_field": "http.user",			            "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.password", 	            "target_field": "http.password",		          "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.proxied", 		            "target_field": "http.proxied",		            "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.orig_fuids",	            "target_field": "log.id.orig_fuids",		      "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.orig_filenames",	        "target_field": "file.orig_filenames",	      "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.orig_mime_types",	      "target_field": "file.orig_mime_types",	      "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.resp_fuids",	            "target_field": "log.id.resp_fuids",		      "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.resp_filenames",	        "target_field": "file.resp_filenames",	      "ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.resp_mime_types",	      "target_field": "file.resp_mime_types",	      "ignore_missing": true 	} },
-    { "rename":   { "field": "message2.stream_id",              "target_field": "http2.stream_id",            "ignore_missing": true  } },
-    { "remove":		{ "field": "message2.tags",		                                                                  "ignore_failure": true } },
-    { "remove":   { "field": ["host"],                                                                            "ignore_failure": true } },
-    { "script":	{ "lang": "painless", "source": "ctx.uri_length = ctx.uri.length()", 			                        "ignore_failure": true 	} },
-    { "script":	{ "lang": "painless", "source": "ctx.useragent_length = ctx.useragent.length()",	                "ignore_failure": true 	} },
-    { "script":	{ "lang": "painless", "source": "ctx.virtual_host_length = ctx.virtual_host.length()", 	          "ignore_failure": true	} },
-    { "pipeline":       { "name": "zeek.common" } }
-  ]
-}
--- a/salt/elasticsearch/files/ingest/zeek.ipsec
+++ b/salt/elasticsearch/files/ingest/zeek.ipsec
@@ -1,38 +0,0 @@
-{
-  "description": "zeek.ipsec",
-  "processors": [
-    {"set": { "field": "event.dataset","value": "ipsec"}},
-    {"json": { "field": "message","target_field": "message2","ignore_failure": true}},
-    {"rename": {"field": "message2.initiator_spi","target_field": "ipsec.initiator_spi","ignore_missing": true}},
-    {"rename": {"field": "message2.responder_spi","target_field": "ipsec.responder_spi","ignore_missing": true}},
-    {"rename": {"field": "message2.maj_ver","target_field": "ipsec.maj_version","ignore_missing": true}},
-    {"rename": {"field": "message2.min_ver","target_field": "ipsec.min_version","ignore_missing": true}},
-    {"set": {"ignore_failure": true,"field": "ipsec.version","value": "{{ipsec.maj_version}}.{{ipsec.min_version}}"}},
-    {"rename": {"field": "message2.exchange_type","target_field": "ipsec.exchange_type","ignore_missing": true}},
-    {"rename": {"field": "message2.flag_e","target_field": "ipsec.flag_e","ignore_missing": true}},
-    {"rename": {"field": "message2.flag_c","target_field": "ipsec.flag_c","ignore_missing": true}},
-    {"rename": {"field": "message2.flag_a","target_field": "ipsec.flag_a","ignore_missing": true}},
-    {"rename": {"field": "message2.flag_i","target_field": "ipsec.flag_i","ignore_missing": true}},
-    {"rename": {"field": "message2.flag_v","target_field": "ipsec.flag_v","ignore_missing": true}},
-    {"rename": {"field": "message2.flag_r","target_field": "ipsec.flag_r","ignore_missing": true}},
-    {"rename": {"field": "message2.message_id","target_field": "ipsec.message_id","ignore_missing": true}},
-    {"rename": {"field": "message2.vendor_ids","target_field": "ipsec.vendor_ids","ignore_missing": true}},
-    {"rename": {"field": "message2.notify_messages","target_field": "ipsec.notify_messages","ignore_missing": true}},
-    {"rename": {"field": "message2.transforms","target_field": "ipsec.transforms","ignore_missing": true}},
-    {"rename": {"field": "message2.ke_dh_groups","target_field": "ipsec.ke_dh_groups","ignore_missing": true}},
-    {"rename": {"field": "message2.proposals","target_field": "ipsec.proposals","ignore_missing": true}},
-    {"rename": {"field": "message2.certificates","target_field": "ipsec.certificates","ignore_missing": true}},
-    {"rename": {"field": "message2.transform_attributes","target_field": "ipsec.transform_attributes","ignore_missing": true}},
-    {"rename": {"field": "message2.length","target_field": "ipsec.length","ignore_missing": true}},
-    {"rename": {"field": "message2.hash","target_field": "ipsec.hash","ignore_missing": true}},
-    {"rename": {"field": "message2.doi","target_field": "ipsec.doi","ignore_missing": true}},
-    {"rename": {"field": "message2.situation","target_field": "ipsec.situation","ignore_missing": true}},
-    {"script": {
-      "lang": "painless",
-      "description": "Remove ipsec fields with empty arrays",
-      "source": "if (ctx.containsKey('ipsec') && ctx.ipsec instanceof Map) {\n        for (String field : ['certificates', 'ke_dh_groups', 'notify_messages', 'proposals', 'transforms', 'transform_attributes', 'vendor_ids']) {\n          if (ctx.ipsec[field] instanceof List && ctx.ipsec[field].isEmpty()) {\n            ctx.ipsec.remove(field);\n          }\n        }\n      }",
-      "ignore_failure": true
-      }},
-    {"pipeline": {"name": "zeek.common"}}
-  ]
-}
--- a/salt/elasticsearch/files/ingest/zeek.ldap
+++ b/salt/elasticsearch/files/ingest/zeek.ldap
@@ -1,25 +0,0 @@
-{
-    "description": "zeek.ldap",
-    "processors": [
-        {"set":      {"field": "event.dataset",                 "value": "ldap"}},
-        {"json":     {"field": "message",                       "target_field": "message2",                     "ignore_failure": true}},
-        {"rename":   {"field": "message2.message_id",           "target_field": "ldap.message_id",              "ignore_missing": true}},
-        {"rename":   {"field": "message2.opcode",               "target_field": "ldap.opcode",                  "ignore_missing": true}},
-        {"rename":   {"field": "message2.result",               "target_field": "ldap.result",                  "ignore_missing": true}},
-        {"rename":   {"field": "message2.diagnostic_message",   "target_field": "ldap.diagnostic_message",      "ignore_missing": true}},
-        {"rename":   {"field": "message2.version",              "target_field": "ldap.version",                 "ignore_missing": true}},
-        {"rename":   {"field": "message2.object",               "target_field": "ldap.object",                  "ignore_missing": true}},
-        {"rename":   {"field": "message2.argument",             "target_field": "ldap.argument",                "ignore_missing": true}},
-        {"rename":   {"field": "message2.scope",                "target_field": "ldap_search.scope",            "ignore_missing":true}},
-        {"rename":   {"field": "message2.deref_aliases",        "target_field": "ldap_search.deref_aliases",    "ignore_missing":true}},
-        {"rename":   {"field": "message2.base_object",          "target_field": "ldap.object",                  "ignore_missing":true}},
-        {"rename":   {"field": "message2.result_count",         "target_field": "ldap_search.result_count",     "ignore_missing":true}},
-        {"rename":   {"field": "message2.filter",               "target_field": "ldap_search.filter",           "ignore_missing":true}},
-        {"rename":   {"field": "message2.attributes",           "target_field": "ldap_search.attributes",       "ignore_missing":true}},
-        {"script":   {"source": "if (ctx.containsKey('ldap') && ctx.ldap.containsKey('diagnostic_message') && ctx.ldap.diagnostic_message != null) {\n          String message = ctx.ldap.diagnostic_message;\n\n          // get user and property from SASL success\n          if (message.toLowerCase().contains(\"sasl(0): successful result\")) {\n            Pattern pattern = /user:\\s*([^ ]+)\\s*property:\\s*([^ ]+)/i;\n            Matcher matcher = pattern.matcher(message);\n            if (matcher.find()) {\n              ctx.ldap.user_email = matcher.group(1);  // Extract user email\n              ctx.ldap.property = matcher.group(2);    // Extract property\n            }\n          }\n          if (message.toLowerCase().contains(\"ldaperr:\")) {\n            Pattern pattern = /comment:\\s*([^,]+)/i;\n            Matcher matcher = pattern.matcher(message);\n\n            if (matcher.find()) {\n              ctx.ldap.comment = matcher.group(1);\n            }\n          }\n        }","ignore_failure": true}},
-        {"script":   {"source": "if (ctx.containsKey('ldap') && ctx.ldap.containsKey('object') && ctx.ldap.object != null) {\n    String message = ctx.ldap.object;\n\n    // parse common name from ldap object\n    if (message.toLowerCase().contains(\"cn=\")) {\n        Pattern pattern = /cn=([^,]+)/i;\n        Matcher matcher = pattern.matcher(message);\n        if (matcher.find()) {\n            ctx.ldap.common_name = matcher.group(1);  // Extract CN\n        }\n    }\n    // build domain from ldap object\n    if (message.toLowerCase().contains(\"dc=\")) {\n        Pattern dcPattern = /dc=([^,]+)/i;\n        Matcher dcMatcher = dcPattern.matcher(message);\n\n        StringBuilder domainBuilder = new StringBuilder();\n        while (dcMatcher.find()) {\n            if (domainBuilder.length() > 0 ){\n                domainBuilder.append(\".\");\n            }\n            domainBuilder.append(dcMatcher.group(1));\n        }\n        if (domainBuilder.length() > 0) {\n            ctx.ldap.domain = domainBuilder.toString();\n        }\n    }\n    // create list of any organizational units from ldap object\n    if (message.toLowerCase().contains(\"ou=\")) {\n        Pattern ouPattern = /ou=([^,]+)/i;\n        Matcher ouMatcher = ouPattern.matcher(message);\n        ctx.ldap.organizational_unit = [];\n\n        while (ouMatcher.find()) {\n            ctx.ldap.organizational_unit.add(ouMatcher.group(1));\n        }\n        if(ctx.ldap.organizational_unit.isEmpty()) {\n          ctx.remove(\"ldap.organizational_unit\");\n        }\n    }\n}\n","ignore_failure": true}},
-        {"remove":   {"field": "message2.tags","ignore_failure": true}},
-        {"remove":   {"field": ["host"],"ignore_failure": true}},
-        {"pipeline": {"name": "zeek.common"}}
-    ]
-}
--- a/salt/elasticsearch/files/ingest/zeek.ldap_search
+++ b/salt/elasticsearch/files/ingest/zeek.ldap_search
@@ -1,9 +0,0 @@
-{
-    "description":"zeek.ldap_search",
-    "processors":[
-        {"pipeline":    {"name": "zeek.ldap",       "ignore_missing_pipeline":true,"ignore_failure":true}},
-        {"set":         {"field": "event.dataset",  "value":"ldap_search"}},
-        {"remove":      {"field": "tags",           "ignore_missing":true}},
-        {"pipeline":    {"name": "zeek.common"}}
-    ]
-}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
m0duspwnens	d91dd0dd3c	watch some values	2024-04-29 17:14:00 -04:00
m0duspwnens	a0388fd568	engines config for valueWatch	2024-04-29 14:02:10 -04:00
m0duspwnens	05244cfd75	watch files change engine	2024-04-24 13:19:39 -04:00
@@ -1 +1 @@
 .4.120
 .4.70