Merge pull request #12659 from Security-Onion-Solutions/2.4/remove-playbook

Remove Playbook ref

.github/.gitleaks.toml

@@ -536,11 +536,10 @@ secretGroup = 4
 
 [allowlist]
 description = "global allow lists"
-regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''']
+regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''']
 paths = [
     '''gitleaks.toml''',
     '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',
     '''(go.mod|go.sum)$''',
-    
     '''salt/nginx/files/enterprise-attack.json'''
 ]

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -0,0 +1,190 @@
+body:
+  - type: markdown
+    attributes:
+      value: |
+        ⚠️ This category is solely for conversations related to Security Onion 2.4 ⚠️
+
+        If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
+  - type: dropdown
+    attributes:
+      label: Version
+      description: Which version of Security Onion 2.4.x are you asking about?
+      options:
+        -
+        - 2.4 Pre-release (Beta, Release Candidate)
+        - 2.4.10
+        - 2.4.20
+        - 2.4.30
+        - 2.4.40
+        - 2.4.50
+        - 2.4.60
+        - 2.4.70
+        - 2.4.80
+        - 2.4.90
+        - 2.4.100
+        - Other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Installation Method
+      description: How did you install Security Onion?
+      options:
+        -
+        - Security Onion ISO image
+        - Network installation on Red Hat derivative like Oracle, Rocky, Alma, etc.
+        - Network installation on Ubuntu
+        - Network installation on Debian
+        - Other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Description
+      description: >
+        Is this discussion about installation, configuration, upgrading, or other?
+      options:
+        -
+        - installation
+        - configuration
+        - upgrading
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Installation Type
+      description: >
+        When you installed, did you choose Import, Eval, Standalone, Distributed, or something else?
+      options:
+        -
+        - Import
+        - Eval
+        - Standalone
+        - Distributed
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Location
+      description: >
+        Is this deployment in the cloud, on-prem with Internet access, or airgap?
+      options:
+        -
+        - cloud
+        - on-prem with Internet access
+        - airgap
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Hardware Specs
+      description: >
+        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://docs.securityonion.net/en/2.4/hardware.html?
+      options:
+        -
+        - Meets minimum requirements
+        - Exceeds minimum requirements
+        - Does not meet minimum requirements
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: CPU
+      description: How many CPU cores do you have?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: RAM
+      description: How much RAM do you have?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Storage for /
+      description: How much storage do you have for the / partition?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Storage for /nsm
+      description: How much storage do you have for the /nsm partition?
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Network Traffic Collection
+      description: >
+        Are you collecting network traffic from a tap or span port?
+      options:
+        -
+        - tap
+        - span port
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Network Traffic Speeds
+      description: >
+        How much network traffic are you monitoring?
+      options:
+        -
+        - Less than 1Gbps
+        - 1Gbps to 10Gbps
+        - more than 10Gbps
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Status
+      description: >
+        Does SOC Grid show all services on all nodes as running OK?
+      options:
+        -
+        - Yes, all services on all nodes are running OK
+        - No, one or more services are failed (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Salt Status
+      description: >
+        Do you get any failures when you run "sudo salt-call state.highstate"?
+      options:
+        -
+        - Yes, there are salt failures (please provide detail below)
+        - No, there are no failures
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Logs
+      description: >
+        Are there any additional clues in /opt/so/log/?
+      options:
+        -
+        - Yes, there are additional clues in /opt/so/log/ (please provide detail below)
+        - No, there are no additional clues
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Detail
+      description: Please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and then provide detailed information to help us help you.
+      placeholder: |-
+        STOP! Before typing, please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 in their entirety!
+
+        If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
+    validations:
+      required: true
+  - type: checkboxes
+    attributes:
+      label: Guidelines
+      options:
+        - label: I have read the discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and assert that I have followed the guidelines.
+          required: true

.github/workflows/close-threads.yml

@@ -0,0 +1,32 @@
+name: 'Close Threads'
+
+on:
+  schedule:
+    - cron: '50 1 * * *'
+  workflow_dispatch:
+
+permissions:
+  issues: write
+  pull-requests: write
+  discussions: write
+
+concurrency:
+  group: lock-threads
+
+jobs:
+  close-threads:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v5
+        with:
+          days-before-issue-stale: -1
+          days-before-issue-close: 60
+          stale-issue-message: "This issue is stale because it has been inactive for an extended period. Stale issues convey that the issue, while important to someone, is not critical enough for the author, or other community members to work on, sponsor, or otherwise shepherd the issue through to a resolution."
+          close-issue-message: "This issue was closed because it has been stale for an extended period. It will be automatically locked in 30 days, after which no further commenting will be available."
+          days-before-pr-stale: 45
+          days-before-pr-close: 60
+          stale-pr-message: "This PR is stale because it has been inactive for an extended period. The longer a PR remains stale the more out of date with the main branch it becomes."
+          close-pr-message: "This PR was closed because it has been stale for an extended period. It will be automatically locked in 30 days. If there is still a commitment to finishing this PR re-open it before it is locked."

.github/workflows/lock-threads.yml

@@ -0,0 +1,25 @@
+name: 'Lock Threads'
+
+on:
+  schedule:
+    - cron: '50 2 * * *'
+  workflow_dispatch:
+
+permissions:
+  issues: write
+  pull-requests: write
+  discussions: write
+
+concurrency:
+  group: lock-threads
+
+jobs:
+  lock-threads:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: jertel/lock-threads@main
+        with:
+          include-discussion-currently-open: true
+          discussion-inactive-days: 90
+          issue-inactive-days: 30
+          pr-inactive-days: 30

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,17 +1,17 @@
-### 2.4.40-20240116 ISO image released on 2024/01/17
+### 2.4.60-20240320 ISO image released on 2024/03/20
 
 
 ### Download and Verify
 
-2.4.40-20240116 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.4.40-20240116.iso
+2.4.60-20240320 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.60-20240320.iso
  
-MD5: AC55D027B663F3CE0878FEBDAD9DD78B  
-SHA1: C2B51723B17F3DC843CC493EB80E93B123E3A3E1  
-SHA256: C5F135FCF45A836BBFF58C231F95E1EA0CD894898322187AD5FBFCD24BC2F123  
+MD5: 178DD42D06B2F32F3870E0C27219821E  
+SHA1: 73EDCD50817A7F6003FE405CF1808A30D034F89D  
+SHA256: DD334B8D7088A7B78160C253B680D645E25984BA5CCAB5CC5C327CA72137FC06  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.40-20240116.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.60-20240320.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
@@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.40-20240116.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.60-20240320.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.4.40-20240116.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.60-20240320.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.4.40-20240116.iso.sig securityonion-2.4.40-20240116.iso
+gpg --verify securityonion-2.4.60-20240320.iso.sig securityonion-2.4.60-20240320.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Tue 16 Jan 2024 07:34:40 PM EST using RSA key ID FE507013
+gpg: Signature made Tue 19 Mar 2024 03:17:58 PM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.4.40
+2.4.70

files/salt/master/master

@@ -41,7 +41,8 @@ file_roots:
   base:
     - /opt/so/saltstack/local/salt
     - /opt/so/saltstack/default/salt
-
+    - /nsm/elastic-fleet/artifacts
+    - /opt/so/rules/nids
 
 # The master_roots setting configures a master-only copy of the file_roots dictionary,
 # used by the state compiler.

pillar/top.sls

@@ -43,8 +43,6 @@ base:
     - soc.soc_soc
     - soc.adv_soc
     - soc.license
-    - soctopus.soc_soctopus
-    - soctopus.adv_soctopus
     - kibana.soc_kibana
     - kibana.adv_kibana
     - kratos.soc_kratos
@@ -61,10 +59,9 @@ base:
     - elastalert.adv_elastalert
     - backup.soc_backup
     - backup.adv_backup
-    - soctopus.soc_soctopus
-    - soctopus.adv_soctopus
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - stig.soc_stig
 
   '*_sensor':
     - healthcheck.sensor
@@ -80,6 +77,8 @@ base:
     - suricata.adv_suricata
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - stig.soc_stig
+    - soc.license
 
   '*_eval':
     - secrets
@@ -105,8 +104,6 @@ base:
     - soc.soc_soc
     - soc.adv_soc
     - soc.license
-    - soctopus.soc_soctopus
-    - soctopus.adv_soctopus
     - kibana.soc_kibana
     - kibana.adv_kibana
     - strelka.soc_strelka
@@ -162,8 +159,6 @@ base:
     - soc.soc_soc
     - soc.adv_soc
     - soc.license
-    - soctopus.soc_soctopus
-    - soctopus.adv_soctopus
     - kibana.soc_kibana
     - kibana.adv_kibana
     - strelka.soc_strelka
@@ -180,6 +175,7 @@ base:
     - suricata.adv_suricata
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - stig.soc_stig
 
   '*_heavynode':
     - elasticsearch.auth
@@ -222,6 +218,8 @@ base:
     - redis.adv_redis
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - stig.soc_stig
+    - soc.license
 
   '*_receiver':
     - logstash.nodes
@@ -256,8 +254,6 @@ base:
     - soc.soc_soc
     - soc.adv_soc
     - soc.license
-    - soctopus.soc_soctopus
-    - soctopus.adv_soctopus
     - kibana.soc_kibana
     - kibana.adv_kibana
     - backup.soc_backup

salt/allowed_states.map.jinja

@@ -34,7 +34,6 @@
           'suricata',
           'utility',
           'schedule',
-          'soctopus',
           'tcpreplay',
           'docker_clean'
           ],
@@ -101,8 +100,8 @@
           'suricata.manager',
           'utility',
           'schedule',
-          'soctopus',
-          'docker_clean'
+          'docker_clean',
+          'stig'
           ],
       'so-managersearch': [
           'salt.master',
@@ -122,8 +121,8 @@
           'suricata.manager',
           'utility',
           'schedule',
-          'soctopus',
-          'docker_clean'
+          'docker_clean',
+          'stig'
           ],
       'so-searchnode': [
           'ssl',
@@ -131,7 +130,8 @@
           'telegraf',
           'firewall',
           'schedule',
-          'docker_clean'
+          'docker_clean',
+          'stig'
           ],
       'so-standalone': [
           'salt.master',
@@ -154,9 +154,9 @@
           'healthcheck',
           'utility',
           'schedule',
-          'soctopus',
           'tcpreplay',
-          'docker_clean'
+          'docker_clean',
+          'stig'
           ],
       'so-sensor': [
           'ssl',
@@ -168,13 +168,15 @@
           'healthcheck',
           'schedule',
           'tcpreplay',
-          'docker_clean'
+          'docker_clean',
+          'stig'
           ],
       'so-fleet': [
           'ssl',
           'telegraf',
           'firewall',
           'logstash',
+          'nginx',
           'healthcheck',
           'schedule',
           'elasticfleet',
@@ -194,10 +196,6 @@
           ],
   }, grain='role') %}
 
-  {% if grains.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone'] %}
-    {% do allowed_states.append('mysql') %}
-  {% endif %}
-
   {%- if grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
     {% do allowed_states.append('zeek') %}
   {%- endif %}
@@ -223,10 +221,6 @@
     {% do allowed_states.append('elastalert') %}
   {% endif %}
 
-  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
-    {% do allowed_states.append('playbook') %}
-  {% endif %}
-
   {% if grains.role in ['so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
     {% do allowed_states.append('logstash') %}
   {% endif %}

salt/bpf/pcap.map.jinja

@@ -1,7 +1,10 @@
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+{% if GLOBALS.pcap_engine == "TRANSITION" %}
+{%   set PCAPBPF = ["ip and host 255.255.255.1 and port 1"] %}
+{% else %}
 {%   import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {%   set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
 {%   import 'bpf/macros.jinja' as MACROS %}
-
 {{   MACROS.remove_comments(BPFMERGED, 'pcap') }}
-
 {%   set PCAPBPF = BPFMERGED.pcap %}
+{% endif %}

salt/bpf/soc_bpf.yaml

@@ -1,6 +1,6 @@
 bpf:
   pcap:
-    description: List of BPF filters to apply to PCAP.
+    description: List of BPF filters to apply to Stenographer.
     multiline: True
     forcedType: "[]string"
     helpLink: bpf.html

salt/common/init.sls

@@ -4,7 +4,6 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
 include:
-  - common.soup_scripts
   - common.packages
 {% if GLOBALS.role in GLOBALS.manager_roles %}
   - manager.elasticsearch # needed for elastic_curl_config state
@@ -134,6 +133,18 @@ common_sbin_jinja:
     - file_mode: 755
     - template: jinja
 
+{% if not GLOBALS.is_manager%}
+# prior to 2.4.50 these scripts were in common/tools/sbin on the manager because of soup and distributed to non managers
+# these two states remove the scripts from non manager nodes
+remove_soup:
+  file.absent:
+    - name: /usr/sbin/soup
+
+remove_so-firewall:
+  file.absent:
+    - name: /usr/sbin/so-firewall
+{% endif %}
+
 so-status_script:
   file.managed:
     - name: /usr/sbin/so-status

salt/common/soup_scripts.sls

@@ -1,23 +1,70 @@
-# Sync some Utilities
-soup_scripts:
-  file.recurse:
-    - name: /usr/sbin
-    - user: root
-    - group: root
-    - file_mode: 755
-    - source: salt://common/tools/sbin
-    - include_pat:
-        - so-common
-        - so-image-common
+{% import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %}
+{% if SOC_GLOBAL.global.airgap %}
+{%   set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %}
+{% else %}
+{%   set UPDATE_DIR='/tmp/sogh/securityonion' %}
+{% endif %}
 
-soup_manager_scripts:
-  file.recurse:
-    - name: /usr/sbin
-    - user: root
-    - group: root
-    - file_mode: 755
-    - source: salt://manager/tools/sbin
-    - include_pat:
-        - so-firewall
-        - so-repo-sync
-        - soup
+remove_common_soup:
+  file.absent:
+    - name: /opt/so/saltstack/default/salt/common/tools/sbin/soup
+
+remove_common_so-firewall:
+  file.absent:
+    - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall
+
+copy_so-common_common_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common
+    - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common
+    - force: True
+    - preserve: True
+
+copy_so-image-common_common_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common
+    - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common
+    - force: True
+    - preserve: True
+
+copy_soup_manager_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/soup
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup
+    - force: True
+    - preserve: True
+
+copy_so-firewall_manager_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-firewall
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
+    - force: True
+    - preserve: True
+
+copy_so-common_sbin:
+  file.copy:
+    - name: /usr/sbin/so-common
+    - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common
+    - force: True
+    - preserve: True
+
+copy_so-image-common_sbin:
+  file.copy:
+    - name: /usr/sbin/so-image-common
+    - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common
+    - force: True
+    - preserve: True
+
+copy_soup_sbin:
+  file.copy:
+    - name: /usr/sbin/soup
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup
+    - force: True
+    - preserve: True
+
+copy_so-firewall_sbin:
+  file.copy:
+    - name: /usr/sbin/so-firewall
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
+    - force: True
+    - preserve: True

salt/common/tools/sbin/so-common

@@ -366,6 +366,13 @@ is_feature_enabled() {
 	return 1
 }
 
+read_feat() {
+	if [ -f /opt/so/log/sostatus/lks_enabled ]; then
+		lic_id=$(cat /opt/so/saltstack/local/pillar/soc/license.sls | grep license_id: | awk '{print $2}')
+		echo "$lic_id/$(cat /opt/so/log/sostatus/lks_enabled)/$(cat /opt/so/log/sostatus/fps_enabled)"
+	fi
+}
+
 require_manager() {
 	if is_manager_node; then
 		echo "This is a manager, so we can proceed."
@@ -559,6 +566,14 @@ status () {
     printf "\n=========================================================================\n$(date) | $1\n=========================================================================\n"
 }
 
+sync_options() {
+	set_version
+	set_os
+	salt_minion_count
+	
+	echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT/$(read_feat)"
+}
+
 systemctl_func() {
 	local action=$1
 	local echo_action=$1

salt/common/tools/sbin/so-common-status-check

@@ -8,6 +8,7 @@
 import sys
 import subprocess
 import os
+import json
 
 sys.path.append('/opt/saltstack/salt/lib/python3.10/site-packages/')
 import salt.config
@@ -36,17 +37,67 @@ def check_needs_restarted():
   with open(outfile, 'w') as f:
     f.write(val)
 
+def check_for_fps():
+    feat = 'fps'
+    feat_full = feat.replace('ps', 'ips')
+    fps = 0
+    try:
+        result = subprocess.run([feat_full + '-mode-setup', '--is-enabled'], stdout=subprocess.PIPE)
+        if result.returncode == 0:
+            fps = 1
+    except FileNotFoundError:
+        fn = '/proc/sys/crypto/' + feat_full + '_enabled'
+        try:
+          with open(fn, 'r') as f:
+              contents = f.read()
+              if '1' in contents:
+                  fps = 1
+        except:
+          # Unknown, so assume 0
+          fps = 0
+
+    with open('/opt/so/log/sostatus/fps_enabled', 'w') as f:
+       f.write(str(fps))
+
+def check_for_lks():
+    feat = 'Lks'
+    feat_full = feat.replace('ks', 'uks')
+    lks = 0
+    result = subprocess.run(['lsblk', '-p', '-J'], check=True, stdout=subprocess.PIPE)
+    data = json.loads(result.stdout)
+    for device in data['blockdevices']:
+        if 'children' in device:
+            for gc in device['children']:
+                if 'children' in gc:
+                    try:
+                        arg = 'is' + feat_full
+                        result = subprocess.run(['cryptsetup', arg, gc['name']], stdout=subprocess.PIPE)
+                        if result.returncode == 0:
+                           lks = 1
+                    except FileNotFoundError:
+                        for ggc in gc['children']:
+                            if 'crypt' in ggc['type']:
+                                lks = 1
+        if lks:
+            break
+    with open('/opt/so/log/sostatus/lks_enabled', 'w') as f:
+       f.write(str(lks))
+
 def fail(msg):
   print(msg, file=sys.stderr)
   sys.exit(1)
 
-
 def main():
   proc = subprocess.run(['id', '-u'], stdout=subprocess.PIPE, encoding="utf-8")
   if proc.stdout.strip() != "0":
     fail("This program must be run as root")
-
+  # Ensure that umask is 0022 so that files created by this script have rw-r-r permissions
+  org_umask = os.umask(0o022)
   check_needs_restarted()
+  check_for_fps()
+  check_for_lks()
+  # Restore umask to whatever value was set before this script was run. SXIG sets to 0077 rw---
+  os.umask(org_umask)
 
 if __name__ == "__main__":
   main()

salt/common/tools/sbin/so-image-common

@@ -53,13 +53,10 @@ container_list() {
       "so-kibana"
       "so-kratos"
       "so-logstash"
-      "so-mysql"
       "so-nginx"
       "so-pcaptools"
-      "so-playbook"
       "so-redis"
       "so-soc"
-      "so-soctopus"
       "so-steno"
       "so-strelka-backend"
       "so-strelka-filestream"

salt/common/tools/sbin/so-ip-update

@@ -49,10 +49,6 @@ if [ "$CONTINUE" == "y" ]; then
 		sed -i "s|$OLD_IP|$NEW_IP|g" $file
 	done
 
-	echo "Granting MySQL root user permissions on $NEW_IP"
-	docker exec -i so-mysql mysql --user=root --password=$(lookup_pillar_secret 'mysql') -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'$NEW_IP' IDENTIFIED BY '$(lookup_pillar_secret 'mysql')' WITH GRANT OPTION;" &> /dev/null
-	echo "Removing MySQL root user from $OLD_IP"
-	docker exec -i so-mysql mysql --user=root --password=$(lookup_pillar_secret 'mysql') -e "DROP USER 'root'@'$OLD_IP';" &> /dev/null
 	echo "Updating Kibana dashboards"
 	salt-call state.apply kibana.so_savedobjects_defaults -l info queue=True
 	

salt/common/tools/sbin/so-log-check

@@ -122,6 +122,7 @@ if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error while communicating"    # Elasticsearch MS -> HN "sensor" temporarily unavailable
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tls handshake error"          # Docker registry container when new node comes onlines
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to get license information" # Logstash trying to contact ES before it's ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process already finished"     # Telegraf script finished just as the auto kill timeout kicked in
 fi
 
 if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
@@ -154,15 +155,11 @@ if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fail\\(error\\)"              # redis/python generic stack line, rely on other lines for actual error
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|urlerror"                     # idstools connection timeout
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeouterror"                 # idstools connection timeout
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|forbidden"                    # playbook
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml"                          # Elastic ML errors 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context canceled"             # elastic agent during shutdown
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|exited with code 128"         # soctopus errors during forced restart by highstate
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|geoip databases update"       # airgap can't update GeoIP DB
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|filenotfounderror"            # bug in 2.4.10 filecheck salt state caused duplicate cronjobs
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|salt-minion-check"            # bug in early 2.4 place Jinja script in non-jinja salt dir causing cron output errors
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|generating elastalert config" # playbook expected error
-    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|activerecord"                 # playbook expected error
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|monitoring.metrics"           # known issue with elastic agent casting the field incorrectly if an integer value shows up before a float
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|repodownload.conf"            # known issue with reposync on pre-2.4.20 
     EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing versions record"      # stenographer corrupt index
@@ -210,7 +207,6 @@ RESULT=0
 CONTAINER_IDS=$(docker ps -q)
 exclude_container so-kibana   # kibana error logs are too verbose with large varieties of errors most of which are temporary
 exclude_container so-idstools # ignore due to known issues and noisy logging
-exclude_container so-playbook # ignore due to several playbook known issues
 
 for container_id in $CONTAINER_IDS; do
     container_name=$(docker ps --format json | jq ". | select(.ID==\"$container_id\")|.Names")

salt/desktop/packages.sls

@@ -334,6 +334,7 @@ desktop_packages:
       - pulseaudio-libs
       - pulseaudio-libs-glib2
       - pulseaudio-utils
+      - putty
       - sane-airscan
       - sane-backends
       - sane-backends-drivers-cameras

salt/docker/defaults.yaml

@@ -67,13 +67,6 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
-    'so-mysql':
-      final_octet: 30
-      port_bindings:
-        - 0.0.0.0:3306:3306
-      custom_bind_mounts: []
-      extra_hosts: []
-      extra_env: []
     'so-nginx':
       final_octet: 31
       port_bindings:
@@ -84,10 +77,10 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
-    'so-playbook':
-      final_octet: 32
+    'so-nginx-fleet-node':
+      final_octet: 31
       port_bindings:
-        - 0.0.0.0:3000:3000
+        - 8443:8443
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
@@ -111,13 +104,6 @@ docker:
       custom_bind_mounts: []
       extra_hosts: []
       extra_env: []
-    'so-soctopus':
-      final_octet: 35
-      port_bindings:
-        - 0.0.0.0:7000:7000
-      custom_bind_mounts: []
-      extra_hosts: []
-      extra_env: []
     'so-strelka-backend':
       final_octet: 36
       custom_bind_mounts: []

salt/docker/soc_docker.yaml

@@ -46,13 +46,11 @@ docker:
     so-kibana: *dockerOptions
     so-kratos: *dockerOptions
     so-logstash: *dockerOptions
-    so-mysql: *dockerOptions
     so-nginx: *dockerOptions
-    so-playbook: *dockerOptions
+    so-nginx-fleet-node: *dockerOptions
     so-redis: *dockerOptions
     so-sensoroni: *dockerOptions
     so-soc: *dockerOptions
-    so-soctopus: *dockerOptions
     so-strelka-backend: *dockerOptions
     so-strelka-filestream: *dockerOptions
     so-strelka-frontend: *dockerOptions

salt/elasticfleet/defaults.yaml

@@ -45,6 +45,8 @@ elasticfleet:
     - cisco_ise
     - cisco_meraki
     - cisco_umbrella
+    - citrix_adc
+    - citrix_waf
     - cloudflare
     - crowdstrike
     - darktrace
@@ -63,6 +65,7 @@ elasticfleet:
     - http_endpoint
     - httpjson
     - iis
+    - journald
     - juniper
     - juniper_srx
     - kafka_log
@@ -75,6 +78,7 @@ elasticfleet:
     - mimecast
     - mysql
     - netflow
+    - nginx
     - o365
     - okta
     - osquery_manager
@@ -103,6 +107,7 @@ elasticfleet:
     - udp
     - vsphere
     - windows
+    - winlog
     - zscaler_zia
     - zscaler_zpa
     - 1password

salt/elasticfleet/enabled.sls

@@ -17,6 +17,11 @@ include:
   - elasticfleet.sostatus
   - ssl
 
+# Wait for Elasticsearch to be ready - no reason to try running Elastic Fleet server if ES is not ready
+wait_for_elasticsearch_elasticfleet:
+  cmd.run:
+    - name: so-elasticsearch-wait
+
 # If enabled, automatically update Fleet Logstash Outputs
 {% if ELASTICFLEETMERGED.config.server.enable_auto_configuration and grains.role not in ['so-import', 'so-eval', 'so-fleet'] %}
 so-elastic-fleet-auto-configure-logstash-outputs:
@@ -33,12 +38,26 @@ so-elastic-fleet-auto-configure-server-urls:
     - retry: True
 {% endif %}
 
-# Automatically update Fleet Server Elasticsearch URLs
+# Automatically update Fleet Server Elasticsearch URLs & Agent Artifact URLs
 {% if grains.role not in ['so-fleet'] %}
 so-elastic-fleet-auto-configure-elasticsearch-urls:
   cmd.run:
     - name: /usr/sbin/so-elastic-fleet-es-url-update
     - retry: True
+
+so-elastic-fleet-auto-configure-artifact-urls:
+  cmd.run:
+    - name: /usr/sbin/so-elastic-fleet-artifacts-url-update
+    - retry: True 
+
+{% endif %}
+
+# Sync Elastic Agent artifacts to Fleet Node
+{% if grains.role in ['so-fleet'] %}
+elasticagent_syncartifacts:
+  file.recurse:
+    - name: /nsm/elastic-fleet/artifacts/beats
+    - source: salt://beats
 {% endif %}
 
 {%   if SERVICETOKEN != '' %}

salt/elasticfleet/files/integrations/grid-nodes_general/rita-logs.json

@@ -0,0 +1,34 @@
+{
+  "package": {
+    "name": "log",
+    "version": ""
+  },
+  "name": "rita-logs",
+  "namespace": "so",
+  "description": "RITA Logs",
+  "policy_id": "so-grid-nodes_general",
+  "vars": {},
+  "inputs": {
+    "logs-logfile": {
+      "enabled": true,
+      "streams": {
+        "log.logs": {
+          "enabled": true,
+          "vars": {
+            "paths": [
+              "/nsm/rita/beacons.csv",
+              "/nsm/rita/exploded-dns.csv",
+              "/nsm/rita/long-connections.csv"
+            ],
+            "exclude_files": [],
+            "ignore_older": "72h",
+            "data_stream.dataset": "rita",
+            "tags": [],
+            "processors": "- dissect:\n    tokenizer: \"/nsm/rita/%{pipeline}.csv\"\n    field: \"log.file.path\"\n    trim_chars: \".csv\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\").split(\"-\");\n          if (pl.length > 1) {\n            pl = pl[1];\n          }\n          else {\n            pl = pl[0];\n          }\n          event.Put(\"@metadata.pipeline\", \"rita.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: rita",
+            "custom": "exclude_lines: ['^Score', '^Source', '^Domain', '^No results']"
+          }
+        }
+      }
+    }
+  }
+}

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-gen-installers

@@ -46,7 +46,7 @@ do
 done 
 
 printf "\n### Stripping out unused components"
-find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -maxdepth 1 -regex '.*fleet.*\|.*packet.*\|.*apm.*\|.*audit.*\|.*heart.*\|.*cloud.*' -delete 
+find /nsm/elastic-agent-workspace/elastic-agent-*/data/elastic-agent-*/components -maxdepth 1 -regex '.*fleet.*\|.*packet.*\|.*apm.*\|.*heart.*\|.*cloud.*' -delete 
 
 printf "\n### Tarring everything up again"
 for OS in "${OSARCH[@]}"

salt/elasticfleet/tools/sbin_jinja/so-elastic-agent-grid-upgrade

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-artifacts-url-update

@@ -0,0 +1,90 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
+# this file except in compliance with the Elastic License 2.0.
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+
+. /usr/sbin/so-common
+
+# Only run on Managers
+if ! is_manager_node; then
+    printf "Not a Manager Node... Exiting"
+    exit 0
+fi
+
+# Function to check if an array contains a value
+array_contains () { 
+    local array="$1[@]"
+    local seeking=$2
+    local in=1
+    for element in "${!array}"; do
+        if [[ $element == "$seeking" ]]; then
+            in=0
+            break
+        fi
+    done
+    return $in
+}
+
+# Query for the current Grid Nodes that are running Logstash (which includes Fleet Nodes)
+LOGSTASHNODES='{{ salt['pillar.get']('logstash:nodes', {}) | tojson }}'
+
+# Initialize an array for new hosts from Fleet Nodes
+declare -a NEW_LIST=()
+
+# Query for Fleet Nodes & add them to the list (Hostname)
+if grep -q "fleet" <<< "$LOGSTASHNODES"; then
+    readarray -t FLEETNODES < <(jq -r '.fleet | keys_unsorted[]' <<< "$LOGSTASHNODES")
+    for NODE in "${FLEETNODES[@]}"; do
+        URL="http://$NODE:8443/artifacts/"
+        NAME="FleetServer_$NODE"
+        NEW_LIST+=("$URL=$NAME")
+    done
+fi
+
+# Create an array for expected hosts and their names
+declare -A expected_urls=(
+    ["http://{{ GLOBALS.url_base }}:8443/artifacts/"]="FleetServer_{{ GLOBALS.hostname }}"
+    ["https://artifacts.elastic.co/downloads/"]="Elastic Artifacts"
+)
+
+# Merge NEW_LIST into expected_urls
+for entry in "${NEW_LIST[@]}"; do
+    # Extract URL and Name from each entry
+    IFS='=' read -r URL NAME <<< "$entry"
+    # Add to expected_urls, automatically handling URL as key and NAME as value
+    expected_urls["$URL"]="$NAME"
+done
+
+# Fetch the current hosts from the API
+current_urls=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/agent_download_sources' | jq -r .items[].host)
+
+# Convert current hosts to an array
+IFS=$'\n' read -rd '' -a current_urls_array <<<"$current_urls"
+
+# Flag to track if any host was added
+any_url_added=0
+
+# Check each expected host
+for host in "${!expected_urls[@]}"; do
+    array_contains current_urls_array "$host" || {
+        echo "$host (${expected_urls[$host]}) is missing. Adding it..."
+        
+        # Prepare the JSON payload
+        JSON_STRING=$( jq -n \
+                        --arg NAME "${expected_urls[$host]}" \
+                        --arg URL "$host" \
+                        '{"name":$NAME,"host":$URL}' )
+        
+        # Create the missing host
+        curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_download_sources" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+        
+        # Flag that an artifact URL was added
+        any_url_added=1
+    }
+
+done
+
+
+if [[ $any_url_added -eq 0 ]]; then
+    echo "All expected artifact URLs are present. No updates needed."
+fi

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-es-url-update

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-outputs-update

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.

salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-urls-update

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
 # or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
 # this file except in compliance with the Elastic License 2.0.

salt/elasticsearch/config.sls

@@ -118,6 +118,19 @@ esingestconf:
     - user: 930
     - group: 939
 
+# Auto-generate Elasticsearch ingest node pipelines from pillar
+{% for pipeline, config in ELASTICSEARCHMERGED.pipelines.items() %}
+es_ingest_conf_{{pipeline}}:
+  file.managed:
+    - name: /opt/so/conf/elasticsearch/ingest/{{ pipeline }}
+    - source: salt://elasticsearch/base-template.json.jinja
+    - defaults:
+      TEMPLATE_CONFIG: {{ config }}
+    - template: jinja
+    - onchanges_in:
+      - file: so-pipelines-reload
+{%     endfor %}
+
 eslog4jfile:
   file.managed:
     - name: /opt/so/conf/elasticsearch/log4j2.properties

salt/elasticsearch/defaults.yaml

@@ -55,6 +55,87 @@ elasticsearch:
             key: /usr/share/elasticsearch/config/elasticsearch.key
             verification_mode: none
   enabled: false
+  pipelines:
+    custom001:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom001
+      - pipeline:
+          name: common
+    custom002:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom002
+      - pipeline:
+          name: common
+    custom003:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom003
+      - pipeline:
+          name: common
+    custom004:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom004
+      - pipeline:
+          name: common
+    custom005:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom005
+      - pipeline:
+          name: common
+    custom006:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom006
+      - pipeline:
+          name: common
+    custom007:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom007
+      - pipeline:
+          name: common
+    custom008:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom008
+      - pipeline:
+          name: common
+    custom009:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom009
+      - pipeline:
+          name: common
+    custom010:
+      description: Custom Pipeline
+      processors:
+      - set:
+          field: tags
+          value: custom010
+      - pipeline:
+          name: common
   index_settings:
     global_overrides:
       index_template:
@@ -117,6 +198,142 @@ elasticsearch:
               sort:
                 field: '@timestamp'
                 order: desc
+    so-detection:
+      index_sorting: false
+      index_template:
+        composed_of:
+        - detection-mappings
+        - detection-settings
+        index_patterns:
+        - so-detection*
+        priority: 500
+        template:
+          mappings:
+            date_detection: false
+            dynamic_templates:
+            - strings_as_keyword:
+                mapping:
+                  ignore_above: 1024
+                  type: keyword
+                match_mapping_type: string
+          settings:
+            index:
+              mapping:
+                total_fields:
+                  limit: 1500
+              number_of_replicas: 0
+              number_of_shards: 1
+              refresh_interval: 30s
+              sort:
+                field: '@timestamp'
+                order: desc
+    so-logs-soc:
+      close: 30
+      delete: 365
+      index_sorting: false
+      index_template:
+        composed_of:
+        - agent-mappings
+        - dtc-agent-mappings
+        - base-mappings
+        - dtc-base-mappings
+        - client-mappings
+        - dtc-client-mappings
+        - container-mappings
+        - destination-mappings
+        - dtc-destination-mappings
+        - pb-override-destination-mappings
+        - dll-mappings
+        - dns-mappings
+        - dtc-dns-mappings
+        - ecs-mappings
+        - dtc-ecs-mappings
+        - error-mappings
+        - event-mappings
+        - dtc-event-mappings
+        - file-mappings
+        - dtc-file-mappings
+        - group-mappings
+        - host-mappings
+        - dtc-host-mappings
+        - http-mappings
+        - dtc-http-mappings
+        - log-mappings
+        - network-mappings
+        - dtc-network-mappings
+        - observer-mappings
+        - dtc-observer-mappings
+        - organization-mappings
+        - package-mappings
+        - process-mappings
+        - dtc-process-mappings
+        - related-mappings
+        - rule-mappings
+        - dtc-rule-mappings
+        - server-mappings
+        - service-mappings
+        - dtc-service-mappings
+        - source-mappings
+        - dtc-source-mappings
+        - pb-override-source-mappings
+        - threat-mappings
+        - tls-mappings
+        - url-mappings
+        - user_agent-mappings
+        - dtc-user_agent-mappings
+        - common-settings
+        - common-dynamic-mappings
+        data_stream: {}
+        index_patterns:
+        - logs-soc-so*
+        priority: 500
+        template:
+          mappings:
+            date_detection: false
+            dynamic_templates:
+            - strings_as_keyword:
+                mapping:
+                  ignore_above: 1024
+                  type: keyword
+                match_mapping_type: string
+          settings:
+            index:
+              lifecycle:
+                name: so-soc-logs
+              mapping:
+                total_fields:
+                  limit: 5000
+              number_of_replicas: 0
+              number_of_shards: 1
+              refresh_interval: 30s
+              sort:
+                field: '@timestamp'
+                order: desc
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+      warm: 7
     so-common:
       close: 30
       delete: 365
@@ -997,6 +1214,50 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-logs-aws_x_cloudfront_logs:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-aws.cloudfront_logs-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-aws.cloudfront_logs-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-aws.cloudfront_logs@package"
+          - "logs-aws.cloudfront_logs@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-aws_x_cloudtrail:
       index_sorting: false
       index_template:
@@ -1217,6 +1478,94 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-logs-aws_x_guardduty:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-aws.guardduty-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-aws.guardduty-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-aws.guardduty@package"
+          - "logs-aws.guardduty@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-logs-aws_x_inspector:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-aws.inspector-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-aws.inspector-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-aws.inspector@package"
+          - "logs-aws.inspector@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-aws_x_route53_public_logs:
       index_sorting: false
       index_template:
@@ -1349,6 +1698,94 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-logs-aws_x_securityhub_findings:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-aws.securityhub_findings-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-aws.securityhub_findings-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-aws.securityhub_findings@package"
+          - "logs-aws.securityhub_findings@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-logs-aws_x_securityhub_insights:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-aws.securityhub_insights-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-aws.securityhub_insights-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-aws.securityhub_insights@package"
+          - "logs-aws.securityhub_insights@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-aws_x_vpcflow:
       index_sorting: false
       index_template:
@@ -2537,6 +2974,270 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-logs-citrix_adc_x_interface:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-citrix_adc.interface-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-citrix_adc.interface-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-citrix_adc.interface@package"
+          - "logs-citrix_adc.interface@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-logs-citrix_adc_x_lbvserver:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-citrix_adc.lbvserver-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-citrix_adc.lbvserver-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-citrix_adc.lbvserver@package"
+          - "logs-citrix_adc.lbvserver@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-logs-citrix_adc_x_service:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-citrix_adc.service-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-citrix_adc.service-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-citrix_adc.service@package"
+          - "logs-citrix_adc.service@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-logs-citrix_adc_x_system:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-citrix_adc.system-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-citrix_adc.system-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-citrix_adc.system@package"
+          - "logs-citrix_adc.system@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-logs-citrix_adc_x_vpn:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-citrix_adc.vpn-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-citrix_adc.vpn-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-citrix_adc.vpn@package"
+          - "logs-citrix_adc.vpn@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-logs-citrix_waf_x_log:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-citrix_waf.log-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-citrix_waf.log-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-citrix_waf.log@package"
+          - "logs-citrix_waf.log@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-cloudflare_x_audit:
       index_sorting: false
       index_template:
@@ -3539,6 +4240,62 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-logs-endpoint_x_diagnostic_x_collection:
+      index_sorting: false
+      index_template:
+        composed_of:
+        - event-mappings
+        - logs-endpoint.diagnostic.collection@custom
+        - logs-endpoint.diagnostic.collection@package
+        - so-fleet_globals-1
+        - so-fleet_agent_id_verification-1
+        data_stream:
+          allow_custom_routing: false
+          hidden: false
+        index_patterns:
+        - .logs-endpoint.diagnostic.collection-*
+        priority: 501
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-endpoint.diagnostic.collection-logs
+              mapping:
+                total_fields:
+                  limit: 5000
+              number_of_replicas: 0
+              sort:
+                field: '@timestamp'
+                order: desc
+      policy:
+        _meta:
+          managed: true
+          managed_by: security_onion
+          package:
+            name: elastic_agent
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-endpoint_x_events_x_api:
       index_sorting: false
       index_template:
@@ -6659,6 +7416,138 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-logs-nginx_x_access:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-nginx.access-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-nginx.access-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-nginx.access@package"
+          - "logs-nginx.access@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-logs-nginx_x_error:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-nginx.error-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-nginx.error-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-nginx.error@package"
+          - "logs-nginx.error@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
+    so-metrics-nginx_x_stubstatus:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "metrics-nginx.stubstatus-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-metrics-nginx.stubstatus-logs
+              number_of_replicas: 0
+        composed_of:
+          - "metrics-nginx.stubstatus@package"
+          - "metrics-nginx.stubstatus@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-o365_x_audit:
       index_sorting: false
       index_template:
@@ -8854,6 +9743,50 @@ elasticsearch:
               set_priority:
                 priority: 50
             min_age: 30d
+    so-logs-winlog_x_winlog:
+      index_sorting: False
+      index_template:
+        index_patterns:
+          - "logs-winlog.winlog-*"
+        template:
+          settings:
+            index:
+              lifecycle:
+                name: so-logs-winlog.winlog-logs
+              number_of_replicas: 0
+        composed_of:
+          - "logs-winlog.winlog@package"
+          - "logs-winlog.winlog@custom"
+          - "so-fleet_globals-1"
+          - "so-fleet_agent_id_verification-1"
+        priority: 501
+        data_stream:
+          hidden: false
+          allow_custom_routing: false
+      policy:
+        phases:
+          cold:
+            actions:
+              set_priority:
+                priority: 0
+            min_age: 30d
+          delete:
+            actions:
+              delete: {}
+            min_age: 365d
+          hot:
+            actions:
+              rollover:
+                max_age: 30d
+                max_primary_shard_size: 50gb
+              set_priority:
+                priority: 100
+            min_age: 0ms
+          warm:
+            actions:
+              set_priority:
+                priority: 50
+            min_age: 30d
     so-logs-zscaler_zia_x_alerts:
       index_sorting: false
       index_template:
@@ -9991,7 +10924,7 @@ elasticsearch:
           hot:
             actions:
               rollover:
-                max_age: 30d
+                max_age: 1d
                 max_primary_shard_size: 50gb
               set_priority:
                 priority: 100

salt/elasticsearch/files/ingest-dynamic/common

@@ -57,10 +57,11 @@
     { "convert":         { "field": "log.id.uid", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "agent.id", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "event.severity", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
-    { "set":             { "field": "event.dataset", "ignore_empty_value":true, "copy_from": "event.dataset_temp"   }},
+    { "set":             { "field": "event.dataset", "ignore_empty_value":true, "copy_from": "event.dataset_temp"   } },
     { "set":             { "if": "ctx.event?.dataset != null && !ctx.event.dataset.contains('.')", "field": "event.dataset", "value": "{{event.module}}.{{event.dataset}}" } },
     { "split":           { "if": "ctx.event?.dataset != null && ctx.event.dataset.contains('.')", "field": "event.dataset", "separator": "\\.", "target_field": "dataset_tag_temp" } },
-    { "append":        { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  }},
+    { "append":          { "if": "ctx.dataset_tag_temp != null", "field": "tags", "value": "{{dataset_tag_temp.1}}"  } },
+    { "grok":            { "if": "ctx.http?.response?.status_code != null", "field": "http.response.status_code", "patterns": ["%{NUMBER:http.response.status_code:long} %{GREEDYDATA}"]} },
     { "remove":          { "field": [ "message2", "type", "fields", "category", "module", "dataset", "dataset_tag_temp", "event.dataset_temp" ], "ignore_missing": true, "ignore_failure": true } }
 {%- endraw %}
 {%- if HIGHLANDER %}

salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0

@@ -0,0 +1,389 @@
+{
+  "description": "Pipeline for pfSense",
+  "processors": [
+    {
+      "set": {
+        "field": "ecs.version",
+        "value": "8.10.0"
+      }
+    },
+    {
+      "set": {
+        "field": "observer.vendor",
+        "value": "netgate"
+      }
+    },
+    {
+      "set": {
+        "field": "observer.type",
+        "value": "firewall"
+      }
+    },
+    {
+      "rename": {
+        "field": "message",
+        "target_field": "event.original"
+      }
+    },
+    {
+      "set": {
+        "field": "event.kind",
+        "value": "event"
+      }
+    },
+    {
+      "set": {
+        "field": "event.timezone",
+        "value": "{{_tmp.tz_offset}}",
+        "if": "ctx._tmp?.tz_offset != null && ctx._tmp?.tz_offset != 'local'"
+      }
+    },
+    {
+      "grok": {
+        "description": "Parse syslog header",
+        "field": "event.original",
+        "patterns": [
+          "^(%{ECS_SYSLOG_PRI})?%{TIMESTAMP} %{GREEDYDATA:message}"
+        ],
+        "pattern_definitions": {
+          "ECS_SYSLOG_PRI": "<%{NONNEGINT:log.syslog.priority:long}>(\\d )?",
+          "TIMESTAMP": "(?:%{BSD_TIMESTAMP_FORMAT}|%{SYSLOG_TIMESTAMP_FORMAT})",
+          "BSD_TIMESTAMP_FORMAT": "%{SYSLOGTIMESTAMP:_tmp.timestamp}(%{SPACE}%{BSD_PROCNAME}|%{SPACE}%{OBSERVER}%{SPACE}%{BSD_PROCNAME})(\\[%{POSINT:process.pid:long}\\])?:",
+          "BSD_PROCNAME": "(?:\\b%{NAME:process.name}|\\(%{NAME:process.name}\\))",
+          "NAME": "[[[:alnum:]]_-]+",
+          "SYSLOG_TIMESTAMP_FORMAT": "%{TIMESTAMP_ISO8601:_tmp.timestamp8601}%{SPACE}%{OBSERVER}%{SPACE}%{PROCESS}%{SPACE}(%{POSINT:process.pid:long}|-) - (-|%{META})",
+          "TIMESTAMP_ISO8601": "%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE:event.timezone}?",
+          "OBSERVER": "(?:%{IP:observer.ip}|%{HOSTNAME:observer.name})",
+          "PROCESS": "(\\(%{DATA:process.name}\\)|(?:%{UNIXPATH}*/)?%{BASEPATH:process.name})",
+          "BASEPATH": "[[[:alnum:]]_%!$@:.,+~-]+",
+          "META": "\\[[^\\]]*\\]"
+        }
+      }
+    },
+    {
+      "date": {
+        "if": "ctx._tmp.timestamp8601 != null",
+        "field": "_tmp.timestamp8601",
+        "target_field": "@timestamp",
+        "formats": [
+          "ISO8601"
+        ]
+      }
+    },
+    {
+      "date": {
+        "if": "ctx.event?.timezone != null && ctx._tmp?.timestamp != null",
+        "field": "_tmp.timestamp",
+        "target_field": "@timestamp",
+        "formats": [
+          "MMM  d HH:mm:ss",
+          "MMM d HH:mm:ss",
+          "MMM dd HH:mm:ss"
+        ],
+        "timezone": "{{ event.timezone }}"
+      }
+    },
+    {
+      "grok": {
+        "description": "Set Event Provider",
+        "field": "process.name",
+        "patterns": [
+          "^%{HYPHENATED_WORDS:event.provider}"
+        ],
+        "pattern_definitions": {
+          "HYPHENATED_WORDS": "\\b[A-Za-z0-9_]+(-[A-Za-z_]+)*\\b"
+        }
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-firewall",
+        "if": "ctx.event.provider == 'filterlog'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-openvpn",
+        "if": "ctx.event.provider == 'openvpn'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-ipsec",
+        "if": "ctx.event.provider == 'charon'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-dhcp",
+        "if": "[\"dhcpd\", \"dhclient\", \"dhcp6c\"].contains(ctx.event.provider)"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-unbound",
+        "if": "ctx.event.provider == 'unbound'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-haproxy",
+        "if": "ctx.event.provider == 'haproxy'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-php-fpm",
+        "if": "ctx.event.provider == 'php-fpm'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-squid",
+        "if": "ctx.event.provider == 'squid'"
+      }
+    },
+    {
+     "pipeline": {
+        "name": "logs-pfsense.log-1.16.0-suricata",
+        "if": "ctx.event.provider == 'suricata'"
+      }
+    },
+    {
+      "drop": {
+        "if": "![\"filterlog\", \"openvpn\", \"charon\", \"dhcpd\", \"dhclient\", \"dhcp6c\", \"unbound\", \"haproxy\", \"php-fpm\", \"squid\", \"suricata\"].contains(ctx.event?.provider)"
+      }
+    },
+    {
+      "append": {
+        "field": "event.category",
+        "value": "network",
+        "if": "ctx.network != null"
+      }
+    },
+    {
+      "convert": {
+        "field": "source.address",
+        "target_field": "source.ip",
+        "type": "ip",
+        "ignore_failure": true,
+        "ignore_missing": true
+      }
+    },
+    {
+      "convert": {
+        "field": "destination.address",
+        "target_field": "destination.ip",
+        "type": "ip",
+        "ignore_failure": true,
+        "ignore_missing": true
+      }
+    },
+    {
+      "set": {
+        "field": "network.type",
+        "value": "ipv6",
+        "if": "ctx.source?.ip != null && ctx.source.ip.contains(\":\")"
+      }
+    },
+    {
+      "set": {
+        "field": "network.type",
+        "value": "ipv4",
+        "if": "ctx.source?.ip != null && ctx.source.ip.contains(\".\")"
+      }
+    },
+    {
+      "geoip": {
+        "field": "source.ip",
+        "target_field": "source.geo",
+        "ignore_missing": true
+      }
+    },
+    {
+      "geoip": {
+        "field": "destination.ip",
+        "target_field": "destination.geo",
+        "ignore_missing": true
+      }
+    },
+    {
+      "geoip": {
+        "ignore_missing": true,
+        "database_file": "GeoLite2-ASN.mmdb",
+        "field": "source.ip",
+        "target_field": "source.as",
+        "properties": [
+          "asn",
+          "organization_name"
+        ]
+      }
+    },
+    {
+      "geoip": {
+        "database_file": "GeoLite2-ASN.mmdb",
+        "field": "destination.ip",
+        "target_field": "destination.as",
+        "properties": [
+          "asn",
+          "organization_name"
+        ],
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "source.as.asn",
+        "target_field": "source.as.number",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "source.as.organization_name",
+        "target_field": "source.as.organization.name",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "destination.as.asn",
+        "target_field": "destination.as.number",
+        "ignore_missing": true
+      }
+    },
+    {
+      "rename": {
+        "field": "destination.as.organization_name",
+        "target_field": "destination.as.organization.name",
+        "ignore_missing": true
+      }
+    },
+    {
+      "community_id": {
+        "target_field": "network.community_id",
+        "ignore_failure": true
+      }
+    },
+    {
+      "grok": {
+        "field": "observer.ingress.interface.name",
+        "patterns": [
+          "%{DATA}.%{NONNEGINT:observer.ingress.vlan.id}"
+        ],
+        "ignore_missing": true,
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "network.vlan.id",
+        "copy_from": "observer.ingress.vlan.id",
+        "ignore_empty_value": true
+      }
+    },
+    {
+      "append": {
+        "field": "related.ip",
+        "value": "{{destination.ip}}",
+        "allow_duplicates": false,
+        "if": "ctx.destination?.ip != null"
+      }
+    },
+    {
+      "append": {
+        "field": "related.ip",
+        "value": "{{source.ip}}",
+        "allow_duplicates": false,
+        "if": "ctx.source?.ip != null"
+      }
+    },
+    {
+      "append": {
+        "field": "related.ip",
+        "value": "{{source.nat.ip}}",
+        "allow_duplicates": false,
+        "if": "ctx.source?.nat?.ip != null"
+      }
+    },
+    {
+      "append": {
+        "field": "related.hosts",
+        "value": "{{destination.domain}}",
+        "if": "ctx.destination?.domain != null"
+      }
+    },
+    {
+      "append": {
+        "field": "related.user",
+        "value": "{{user.name}}",
+        "if": "ctx.user?.name != null"
+      }
+    },
+    {
+      "set": {
+        "field": "network.direction",
+        "value": "{{network.direction}}bound",
+        "if": "ctx.network?.direction != null && ctx.network?.direction =~ /^(in|out)$/"
+      }
+    },
+    {
+      "remove": {
+        "field": [
+          "_tmp"
+        ],
+        "ignore_failure": true
+      }
+    },
+    {
+      "script": {
+        "lang": "painless",
+        "description": "This script processor iterates over the whole document to remove fields with null values.",
+        "source": "void handleMap(Map map) {\n  for (def x : map.values()) {\n    if (x instanceof Map) {\n        handleMap(x);\n    } else if (x instanceof List) {\n        handleList(x);\n    }\n  }\n  map.values().removeIf(v -> v == null || (v instanceof String && v == \"-\"));\n}\nvoid handleList(List list) {\n  for (def x : list) {\n      if (x instanceof Map) {\n          handleMap(x);\n      } else if (x instanceof List) {\n          handleList(x);\n      }\n  }\n}\nhandleMap(ctx);\n"
+      }
+    },
+    {
+      "remove": {
+        "field": "event.original",
+        "if": "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))",
+        "ignore_failure": true,
+        "ignore_missing": true
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-pfsense.log@custom",
+        "ignore_missing_pipeline": true
+      }
+    }
+  ],
+  "on_failure": [
+    {
+      "remove": {
+        "field": [
+          "_tmp"
+        ],
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "event.kind",
+        "value": "pipeline_error"
+      }
+    },
+    {
+      "append": {
+        "field": "error.message",
+        "value": "{{{ _ingest.on_failure_message }}}"
+      }
+    }
+  ],
+  "_meta": {
+    "managed_by": "fleet",
+    "managed": true,
+    "package": {
+      "name": "pfsense"
+    }
+  }
+}

salt/elasticsearch/files/ingest/logs-pfsense.log-1.16.0-suricata

@@ -0,0 +1,31 @@
+{
+  "description": "Pipeline for parsing pfSense Suricata logs.",
+  "processors": [
+    {
+     "pipeline": {
+        "name": "suricata.common"
+      }
+    }
+  ],
+  "on_failure": [
+    {
+      "set": {
+        "field": "event.kind",
+        "value": "pipeline_error"
+      }
+    },
+    {
+      "append": {
+        "field": "error.message",
+        "value": "{{{ _ingest.on_failure_message }}}"
+      }
+    }
+  ],
+  "_meta": {
+    "managed_by": "fleet",
+    "managed": true,
+    "package": {
+      "name": "pfsense"
+    }
+  }
+}

salt/elasticsearch/files/ingest/strelka.file

@@ -67,6 +67,7 @@
     { "set":         { "if": "ctx.scan?.pe?.image_version == '0'",   "field": "scan.pe.image_version", "value": "0.0", "override": true }  },
     { "set":         { "field": "observer.name",        "value": "{{agent.name}}" }},
     { "convert" :    { "field" : "scan.exiftool","type": "string", "ignore_missing":true }},
+    { "convert" :    { "field" : "scan.pe.flags","type": "string", "ignore_missing":true }},
     { "remove":      { "field": ["host", "path", "message", "exiftool", "scan.yara.meta"],                                         "ignore_missing": true  } },
     { "pipeline":    { "name": "common"                                                                                   } }
   ]

salt/elasticsearch/files/ingest/suricata.common

@@ -4,6 +4,7 @@
     { "json":   { "field": "message",                   "target_field": "message2",             "ignore_failure": true  } },
     { "rename": { "field": "message2.pkt_src",          "target_field": "network.packet_source","ignore_failure": true  } },
     { "rename": { "field": "message2.proto",            "target_field": "network.transport",    "ignore_failure": true  } },
+    { "rename": { "field": "message2.in_iface",         "target_field": "observer.ingress.interface.name",    "ignore_failure": true  } },
     { "rename": { "field": "message2.flow_id",          "target_field": "log.id.uid",           "ignore_failure": true  } },
     { "rename": { "field": "message2.src_ip",           "target_field": "source.ip",            "ignore_failure": true  } },
     { "rename": { "field": "message2.src_port",         "target_field": "source.port",          "ignore_failure": true  } },

salt/elasticsearch/files/ingest/suricata.ike

@@ -0,0 +1,21 @@
+{
+  "description" : "suricata.ike",
+  "processors" : [
+    { "rename": 	{ "field": "message2.proto", 			"target_field": "network.transport",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
+    { "rename": 	{ "field": "message2.ike.alg_auth", 		"target_field": "ike.algorithm.authentication", "ignore_missing": true 	} },
+    { "rename":         { "field": "message2.ike.alg_enc",              "target_field": "ike.algorithm.encryption",     "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ike.alg_esn",              "target_field": "ike.algorithm.esn",            "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ike.alg_dh",               "target_field": "ike.algorithm.dh",             "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ike.alg_prf",              "target_field": "ike.algorithm.prf",            "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ike.exchange_type",        "target_field": "ike.exchange_type",            "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ike.payload",              "target_field": "ike.payload",                  "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ike.role",                 "target_field": "ike.role",                     "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ike.init_spi",             "target_field": "ike.spi.initiator",            "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ike.resp_spi",             "target_field": "ike.spi.responder",            "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ike.version_major",        "target_field": "ike.version.major",            "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ike.version_minor",        "target_field": "ike.version.minor",            "ignore_missing": true  } },
+    { "rename":         { "field": "message2.ike.ikev2.errors",         "target_field": "ike.ikev2.errors",             "ignore_missing": true  } },
+    { "pipeline": { "name": "common" } }
+  ]
+}

salt/elasticsearch/files/ingest/suricata.ikev2

@@ -1,8 +0,0 @@
-{
-  "description" : "suricata.ikev2",
-  "processors" : [
-    { "rename": 	{ "field": "message2.proto", 		"target_field": "network.transport",		"ignore_missing": true 	} },
-    { "rename": 	{ "field": "message2.app_proto", 		"target_field": "network.protocol",		"ignore_missing": true 	} },
-    { "pipeline": { "name": "common" } }
-  ]
-}

salt/elasticsearch/soc_elasticsearch.yaml

@@ -45,6 +45,28 @@ elasticsearch:
             description: Max number of boolean clauses per query.
             global: True
             helpLink: elasticsearch.html
+  pipelines:
+    custom001: &pipelines
+      description:
+        description: Description of the ingest node pipeline
+        global: True
+        advanced: True
+        helpLink: elasticsearch.html
+      processors:
+        description: Processors for the ingest node pipeline
+        global: True
+        advanced: True
+        multiline: True
+        helpLink: elasticsearch.html
+    custom002: *pipelines
+    custom003: *pipelines
+    custom004: *pipelines
+    custom005: *pipelines
+    custom006: *pipelines
+    custom007: *pipelines
+    custom008: *pipelines
+    custom009: *pipelines
+    custom010: *pipelines
   index_settings:
     global_overrides:
       index_template:
@@ -73,6 +95,7 @@ elasticsearch:
                   description: The order to sort by. Must set index_sorting to True.
                   global: True
                   helpLink: elasticsearch.html
+      policy:
         phases:
           hot:
             max_age:
@@ -318,6 +341,7 @@ elasticsearch:
     so-logs-windows_x_powershell: *indexSettings
     so-logs-windows_x_powershell_operational: *indexSettings
     so-logs-windows_x_sysmon_operational: *indexSettings
+    so-logs-winlog_x_winlog: *indexSettings
     so-logs-apache_x_access: *indexSettings
     so-logs-apache_x_error: *indexSettings
     so-logs-auditd_x_log: *indexSettings
@@ -346,6 +370,12 @@ elasticsearch:
     so-logs-cisco_ftd_x_log: *indexSettings
     so-logs-cisco_ios_x_log: *indexSettings
     so-logs-cisco_ise_x_log: *indexSettings
+    so-logs-citrix_adc_x_interface: *indexSettings
+    so-logs-citrix_adc_x_lbvserver: *indexSettings
+    so-logs-citrix_adc_x_service: *indexSettings
+    so-logs-citrix_adc_x_system: *indexSettings
+    so-logs-citrix_adc_x_vpn: *indexSettings
+    so-logs-citrix_waf_x_log: *indexSettings
     so-logs-cloudflare_x_audit: *indexSettings
     so-logs-cloudflare_x_logpull: *indexSettings
     so-logs-crowdstrike_x_falcon: *indexSettings
@@ -406,6 +436,8 @@ elasticsearch:
     so-logs-mysql_x_error: *indexSettings
     so-logs-mysql_x_slowlog: *indexSettings
     so-logs-netflow_x_log: *indexSettings
+    so-logs-nginx_x_access: *indexSettings
+    so-logs-nginx_x_error: *indexSettings
     so-logs-o365_x_audit: *indexSettings
     so-logs-okta_x_system: *indexSettings
     so-logs-panw_x_panos: *indexSettings
@@ -471,6 +503,7 @@ elasticsearch:
     so-metrics-endpoint_x_metadata: *indexSettings
     so-metrics-endpoint_x_metrics: *indexSettings
     so-metrics-endpoint_x_policy: *indexSettings
+    so-metrics-nginx_x_stubstatus: *indexSettings
     so-case: *indexSettings
     so-common: *indexSettings
     so-endgame: *indexSettings

salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json

@@ -1,4 +1,5 @@
-        {"template": {
+{
+  "template": {
     "settings": {
       "index": {
         "lifecycle": {
@@ -379,4 +380,4 @@
     "managed_by": "fleet",
     "managed": true
   }
-      }
+}

salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@custom.json

@@ -0,0 +1,12 @@
+{
+  "template": {
+    "settings": {}
+  },
+  "_meta": {
+    "package": {
+      "name": "endpoint"
+    },
+    "managed_by": "fleet",
+    "managed": true
+  }
+}

salt/elasticsearch/templates/component/elastic-agent/logs-endpoint.diagnostic.collection@package.json

@@ -0,0 +1,132 @@
+{
+  "template": {
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": "logs-endpoint.collection-diagnostic"
+        },
+        "codec": "best_compression",
+        "default_pipeline": "logs-endpoint.diagnostic.collection-8.10.2",
+        "mapping": {
+          "total_fields": {
+            "limit": "10000"
+          },
+          "ignore_malformed": "true"
+        },
+        "query": {
+          "default_field": [
+            "ecs.version",
+            "event.action",
+            "event.category",
+            "event.code",
+            "event.dataset",
+            "event.hash",
+            "event.id",
+            "event.kind",
+            "event.module",
+            "event.outcome",
+            "event.provider",
+            "event.type"
+          ]
+        }
+      }
+    },
+    "mappings": {
+      "dynamic": false,
+      "properties": {
+        "@timestamp": {
+          "ignore_malformed": false,
+          "type": "date"
+        },
+        "ecs": {
+          "properties": {
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        },
+        "data_stream": {
+          "properties": {
+            "namespace": {
+              "type": "constant_keyword"
+            },
+            "type": {
+              "type": "constant_keyword"
+            },
+            "dataset": {
+              "type": "constant_keyword"
+            }
+          }
+        },
+        "event": {
+          "properties": {
+            "severity": {
+              "type": "long"
+            },
+            "code": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "created": {
+              "type": "date"
+            },
+            "kind": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "module": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "type": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "sequence": {
+              "type": "long"
+            },
+            "ingested": {
+              "type": "date"
+            },
+            "provider": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "action": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "id": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "category": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "dataset": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "hash": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "outcome": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        }
+      }
+    }
+  },
+  "_meta": {
+    "package": {
+      "name": "endpoint"
+    },
+    "managed_by": "fleet",
+    "managed": true
+  }
+}

salt/elasticsearch/templates/component/elastic-agent/logs-system.syslog@custom.json

@@ -0,0 +1,22 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "error": {
+          "properties": {
+            "message": {
+              "type": "match_only_text"
+            }
+	  }
+	}
+      }
+    }
+  },
+  "_meta": {
+    "package": {
+      "name": "system"
+    },
+    "managed_by": "fleet",
+    "managed": true
+  }
+}

salt/elasticsearch/templates/component/so/detection-mappings.json

@@ -0,0 +1,138 @@
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "so_audit_doc_id": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "@timestamp": {
+          "type": "date"
+        },
+        "so_kind": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "so_operation": {
+          "ignore_above": 1024,
+          "type": "keyword"
+        },
+        "so_detection": {
+          "properties": {
+            "publicId": {
+              "type": "text"
+            },
+            "title": {
+              "type": "text"
+            },
+            "severity": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "author": {
+              "type": "text"
+            },
+            "description": {
+              "type": "text"
+            },
+            "content": {
+              "type": "text"
+            },
+            "isEnabled": {
+              "type": "boolean"
+            },
+            "isReporting": {
+              "type": "boolean"
+            },
+            "isCommunity": {
+              "type": "boolean"
+            },
+            "tags": {
+              "type": "text"
+            },
+            "ruleset": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "engine": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "language": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "license": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "overrides": {
+              "properties": {
+                "type": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "isEnabled": {
+                  "type": "boolean"
+                },
+                "createdAt": {
+                  "type": "date"
+                },
+                "updatedAt": {
+                  "type": "date"
+                },
+                "regex": {
+                  "type": "text"
+                },
+                "value": {
+                  "type": "text"
+                },
+                "thresholdType": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "track": {
+                  "ignore_above": 1024,
+                  "type": "keyword"
+                },
+                "ip": {
+                  "type": "text"
+                },
+                "count": {
+                  "type": "long"
+                },
+                "seconds": {
+                  "type": "long"
+                },
+                "customFilter": {
+                  "type": "text"
+                }
+              }
+            }
+          }
+        },
+        "so_detectioncomment": {
+          "properties": {
+            "createTime": {
+              "type": "date"
+            },
+            "detectionId": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "value": {
+              "type": "text"
+            },
+            "userId": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        }
+      }
+    }
+  },
+  "_meta": {
+    "ecs_version": "1.12.2"
+  }
+}

salt/elasticsearch/templates/component/so/detection-settings.json

@@ -0,0 +1,7 @@
+{
+  "template": {},
+  "version": 1,
+  "_meta": {
+    "description": "default settings for common Security Onion Detections indices"
+  }
+}

salt/elasticsearch/templates/component/so/so-scan-mappings.json

@@ -14,15 +14,18 @@
             },
             "pe": {
               "properties": {
+                "flags": {
+                  "type": "text"
+                },
+		"image_version": {
+                  "type": "float"
+                },
 		"sections": {
                   "properties": {
                     "entropy": {
                       "type": "float"
                     }
                   }
-                },
-		"image_version": {
-	          "type": "float"
                 }
               }
             },

salt/firewall/containers.map.jinja

@@ -9,11 +9,9 @@
      'so-influxdb',
      'so-kibana',
      'so-kratos',
-     'so-mysql',
      'so-nginx',
      'so-redis',
      'so-soc',
-     'so-soctopus',
      'so-strelka-coordinator',
      'so-strelka-gatekeeper',
      'so-strelka-frontend',
@@ -32,11 +30,9 @@
      'so-kibana',
      'so-kratos',
      'so-logstash',
-     'so-mysql',
      'so-nginx',
      'so-redis',
      'so-soc',
-     'so-soctopus',
      'so-strelka-coordinator',
      'so-strelka-gatekeeper',
      'so-strelka-frontend',
@@ -95,6 +91,7 @@
 {% set NODE_CONTAINERS = [
      'so-elastic-fleet',
      'so-logstash',
+     'so-nginx-fleet-node'
 ] %}
 
 {% elif GLOBALS.role == 'so-sensor' %}

salt/firewall/defaults.yaml

@@ -98,19 +98,11 @@ firewall:
       tcp: 
         - 7788
       udp: []
-    mysql:
-      tcp:
-        - 3306
-      udp: []
     nginx:
       tcp:
         - 80
         - 443
       udp: []
-    playbook:
-      tcp:
-        - 3000
-      udp: []
     redis:
       tcp:
         - 6379
@@ -178,8 +170,6 @@ firewall:
           hostgroups:
             eval:
               portgroups:
-                - playbook
-                - mysql
                 - kibana
                 - redis
                 - influxdb
@@ -363,8 +353,6 @@ firewall:
           hostgroups:
             manager:
               portgroups:
-                - playbook
-                - mysql
                 - kibana
                 - redis
                 - influxdb
@@ -559,8 +547,6 @@ firewall:
           hostgroups:
             managersearch:
               portgroups:
-                - playbook
-                - mysql
                 - kibana
                 - redis
                 - influxdb
@@ -756,8 +742,6 @@ firewall:
                 - all
             standalone:
               portgroups:
-                - playbook
-                - mysql
                 - kibana
                 - redis
                 - influxdb
@@ -1295,6 +1279,10 @@ firewall:
               portgroups:
                 - redis
                 - beats_5644
+            managersearch:
+              portgroups:
+                - redis
+                - beats_5644
             self:
               portgroups:
                 - redis

salt/firewall/soc_firewall.yaml

@@ -121,15 +121,9 @@ firewall:
     localrules:
       tcp: *tcpsettings
       udp: *udpsettings
-    mysql:
-      tcp: *tcpsettings
-      udp: *udpsettings
     nginx:
       tcp: *tcpsettings
       udp: *udpsettings
-    playbook:
-      tcp: *tcpsettings
-      udp: *udpsettings
     redis:
       tcp: *tcpsettings
       udp: *udpsettings

salt/global/defaults.yaml

@@ -0,0 +1,2 @@
+global:
+  pcapengine: STENO

salt/global/map.jinja

@@ -0,0 +1,2 @@
+{% import_yaml 'global/defaults.yaml' as GLOBALDEFAULTS %}
+{% set GLOBALMERGED = salt['pillar.get']('global', GLOBALDEFAULTS.global, merge=True) %}

salt/global/soc_global.yaml

@@ -10,10 +10,15 @@ global:
     regex: ^(([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?)?$
     regexFailureMessage: You must enter a valid IP address or CIDR.
   mdengine:
-    description: What engine to use for meta data generation. Options are ZEEK and SURICATA.
+    description: Which engine to use for meta data generation. Options are ZEEK and SURICATA.
     regex: ^(ZEEK|SURICATA)$
     regexFailureMessage: You must enter either ZEEK or SURICATA.
     global: True
+  pcapengine:
+    description: Which engine to use for generating pcap. Options are STENO, SURICATA or TRANSITION.
+    regex: ^(STENO|SURICATA|TRANSITION)$
+    regexFailureMessage: You must enter either STENO, SURICATA or TRANSITION.
+    global: True
   ids:
     description: Which IDS engine to use. Currently only Suricata is supported.
     global: True

salt/idstools/enabled.sls

@@ -39,7 +39,7 @@ so-idstools:
     {% endif %}
     - binds:
       - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro
-      - /opt/so/rules/nids:/opt/so/rules/nids:rw
+      - /opt/so/rules/nids/suri:/opt/so/rules/nids/suri:rw
       - /nsm/rules/:/nsm/rules/:rw
     {% if DOCKER.containers['so-idstools'].custom_bind_mounts %}
       {% for BIND in DOCKER.containers['so-idstools'].custom_bind_mounts %}

salt/idstools/etc/rulecat.conf

@@ -1,10 +1,10 @@
 {%- from 'vars/globals.map.jinja' import GLOBALS -%}
 {%- from 'idstools/map.jinja' import IDSTOOLSMERGED -%}
---merged=/opt/so/rules/nids/all.rules
---local=/opt/so/rules/nids/local.rules
+--merged=/opt/so/rules/nids/suri/all.rules
+--local=/opt/so/rules/nids/suri/local.rules
 {%-   if GLOBALS.md_engine == "SURICATA" %}
---local=/opt/so/rules/nids/extraction.rules
---local=/opt/so/rules/nids/filters.rules
+--local=/opt/so/rules/nids/suri/extraction.rules
+--local=/opt/so/rules/nids/suri/filters.rules
 {%-   endif %}
 --url=http://{{ GLOBALS.manager }}:7788/suricata/emerging-all.rules
 --disable=/opt/so/idstools/etc/disable.conf

salt/idstools/soc_idstools.yaml

@@ -6,9 +6,10 @@ idstools:
       description: Enter your registration code or oinkcode for paid NIDS rulesets.
       title: Registration Code
       global: True
+      forcedType: string
       helpLink: rules.html
     ruleset:
-      description: Defines the ruleset you want to run. Options are ETOPEN or ETPRO.
+      description: 'Defines the ruleset you want to run. Options are ETOPEN or ETPRO. WARNING! Changing the ruleset will remove all existing Suricata rules of the previous ruleset and their associated overrides. This removal cannot be undone.'
       global: True
       regex:  ETPRO\b|ETOPEN\b
       helpLink: rules.html

salt/idstools/sync_files.sls

@@ -21,7 +21,7 @@ idstoolsetcsync:
 
 rulesdir:
   file.directory:
-    - name: /opt/so/rules/nids
+    - name: /opt/so/rules/nids/suri
     - user: 939
     - group: 939
     - makedirs: True
@@ -29,7 +29,7 @@ rulesdir:
 # Don't show changes because all.rules can be large
 synclocalnidsrules:
   file.recurse:
-    - name: /opt/so/rules/nids/
+    - name: /opt/so/rules/nids/suri/
     - source: salt://idstools/rules/
     - user: 939
     - group: 939

salt/kratos/map.jinja

@@ -21,7 +21,7 @@
 
 {% set KRATOSMERGED = salt['pillar.get']('kratos', default=KRATOSDEFAULTS.kratos, merge=true) %}
 
-{% if KRATOSMERGED.oidc.enabled and 'oidc' in salt['pillar.get']('features') %}
+{% if KRATOSMERGED.oidc.enabled and 'odc' in salt['pillar.get']('features') %}
 {%   do KRATOSMERGED.config.selfservice.methods.update({'oidc': {'enabled': true, 'config': {'providers': [KRATOSMERGED.oidc.config]}}}) %}
 {% endif %}
 

salt/logstash/config.sls

@@ -63,6 +63,20 @@ lspipelinedir:
     - user: 931
     - group: 939
 
+# Auto-generate Logstash pipeline config
+{% for pipeline, config in LOGSTASH_MERGED.pipeline_config.items() %}
+{% for assigned_pipeline in ASSIGNED_PIPELINES %}
+{% set custom_pipeline = 'custom/' + pipeline + '.conf' %}
+{% if custom_pipeline in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
+ls_custom_pipeline_conf_{{assigned_pipeline}}_{{pipeline}}:
+  file.managed:
+    - name: /opt/so/conf/logstash/pipelines/{{assigned_pipeline}}/{{ pipeline }}.conf
+    - contents: LOGSTASH_MERGED.pipeline_config.{{pipeline}}
+{% endif %}
+{% endfor %}
+{% endfor %}
+
+
 {% for assigned_pipeline in ASSIGNED_PIPELINES %}
     {% for CONFIGFILE in LOGSTASH_MERGED.defined_pipelines[assigned_pipeline] %}
 ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }}:

salt/logstash/defaults.yaml

@@ -42,6 +42,24 @@ logstash:
     custom2: []
     custom3: []
     custom4: []
+  pipeline_config:
+    custom001: |-
+      filter {
+        if [event][module] =~ "zeek" {
+          mutate {
+            add_tag => ["network_stuff"]
+          }
+        }
+      }
+    custom002: PLACEHOLDER
+    custom003: PLACEHOLDER
+    custom004: PLACEHOLDER
+    custom005: PLACEHOLDER
+    custom006: PLACEHOLDER
+    custom007: PLACEHOLDER
+    custom008: PLACEHOLDER
+    custom009: PLACEHOLDER
+    custom010: PLACEHOLDER
   settings:
     lsheap: 500m
   config:

salt/logstash/soc_logstash.yaml

@@ -31,6 +31,22 @@ logstash:
     custom2: *defined_pipelines
     custom3: *defined_pipelines
     custom4: *defined_pipelines
+  pipeline_config:
+    custom001: &pipeline_config
+      description: Pipeline configuration for Logstash
+      advanced: True
+      multiline: True
+      forcedType: string
+      helpLink: logstash.html
+    custom002: *pipeline_config
+    custom003: *pipeline_config
+    custom004: *pipeline_config
+    custom005: *pipeline_config
+    custom006: *pipeline_config
+    custom007: *pipeline_config
+    custom008: *pipeline_config
+    custom009: *pipeline_config
+    custom010: *pipeline_config
   settings:
     lsheap: 
       description: Heap size to use for logstash

salt/manager/files/mirror.txt

@@ -0,0 +1,2 @@
+https://repo.securityonion.net/file/so-repo/prod/2.4/oracle/9
+https://repo-alt.securityonion.net/prod/2.4/oracle/9

salt/manager/files/repodownload.conf

@@ -0,0 +1,13 @@
+[main]
+gpgcheck=1
+installonly_limit=3
+clean_requirements_on_remove=True
+best=True
+skip_if_unavailable=False
+cachedir=/opt/so/conf/reposync/cache
+keepcache=0
+[securityonionsync]
+name=Security Onion Repo repo
+mirrorlist=file:///opt/so/conf/reposync/mirror.txt
+enabled=1
+gpgcheck=1

salt/manager/init.sls

@@ -75,6 +75,20 @@ yara_update_scripts:
     - defaults:
         EXCLUDEDRULES: {{ STRELKAMERGED.rules.excluded }}
 
+so-repo-file:
+  file.managed:
+    - name: /opt/so/conf/reposync/repodownload.conf
+    - source: salt://manager/files/repodownload.conf
+    - user: socore
+    - group: socore
+
+so-repo-mirrorlist:
+  file.managed:
+    - name: /opt/so/conf/reposync/mirror.txt
+    - source: salt://manager/files/mirror.txt
+    - user: socore
+    - group: socore
+
 so-repo-sync:
   {%     if MANAGERMERGED.reposync.enabled %}
   cron.present:
@@ -103,55 +117,6 @@ rules_dir:
     - group: socore
     - makedirs: True
 
-{%    if STRELKAMERGED.rules.enabled %}
-
-strelkarepos:
-  file.managed:
-    - name: /opt/so/conf/strelka/repos.txt
-    - source: salt://strelka/rules/repos.txt.jinja
-    - template: jinja
-    - defaults:
-        STRELKAREPOS: {{ STRELKAMERGED.rules.repos }}
-    - makedirs: True
-
-strelka-yara-update:
-  {%       if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %} 
-  cron.present:
-  {%       else %}
-  cron.absent:
-  {%       endif %}
-    - user: socore
-    - name: '/usr/sbin/so-yara-update >> /opt/so/log/yarasync/yara-update.log 2>&1'
-    - identifier: strelka-yara-update
-    - hour: '7'
-    - minute: '1'
-
-strelka-yara-download:
-  {%       if MANAGERMERGED.reposync.enabled and not GLOBALS.airgap %}
-  cron.present:
-  {%       else %}
-  cron.absent:
-  {%       endif %}
-    - user: socore
-    - name: '/usr/sbin/so-yara-download >> /opt/so/log/yarasync/yara-download.log 2>&1'
-    - identifier: strelka-yara-download
-    - hour: '7'
-    - minute: '1'
-
-{%      if not GLOBALS.airgap %}
-update_yara_rules:
-  cmd.run:
-    - name: /usr/sbin/so-yara-update
-    - onchanges:
-      - file: yara_update_scripts
-
-download_yara_rules:
-  cmd.run:
-    - name: /usr/sbin/so-yara-download
-    - onchanges:
-      - file: yara_update_scripts
-{%      endif %}
-{%     endif %}
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/manager/soc_manager.yaml

@@ -20,10 +20,6 @@ manager:
     description: String of hosts to ignore the proxy settings for. 
     global: True
     helpLink: proxy.html
-  playbook:
-    description: Enable playbook 1=enabled 0=disabled.
-    global: True
-    helpLink: playbook.html
   proxy:
     description: Proxy server to use for updates.
     global: True

salt/manager/tools/sbin/so-minion

@@ -79,6 +79,32 @@ function getinstallinfo() {
 	source <(echo $INSTALLVARS)
 }
 
+function pcapspace() {
+	if [[ "$OPERATION" == "setup" ]]; then
+		# Use 25% for PCAP
+		PCAP_PERCENTAGE=1
+		DFREEPERCENT=21
+		local SPACESIZE=$(df -k /nsm | tail -1 | awk '{print $2}' | tr -d \n)
+	else
+
+		local NSMSIZE=$(salt "$MINION_ID" disk.usage --out=json | jq -r '.[]."/nsm"."1K-blocks" ')
+		local ROOTSIZE=$(salt "$MINION_ID" disk.usage --out=json | jq -r '.[]."/"."1K-blocks" ')
+	
+		if [[ "$NSMSIZE" == "null" ]]; then
+			# Looks like there is no dedicated nsm partition. Using root
+			local SPACESIZE=$ROOTSIZE
+		else
+			local SPACESIZE=$NSMSIZE
+		fi
+	fi
+
+	local s=$(( $SPACESIZE / 1000000 ))
+	local s1=$(( $s / 4 * $PCAP_PERCENTAGE ))
+
+	MAX_PCAP_SPACE=$s1
+
+}
+
 function testMinion() {
 	# Always run on the host, since this is going to be the manager of a distributed grid, or an eval/standalone.
 	# Distributed managers must run this in order for the sensor nodes to have access to the so-tcpreplay image.
@@ -244,6 +270,10 @@ function add_sensor_to_minion() {
 	echo "      lb_procs: '$CORECOUNT'" >> $PILLARFILE
 	echo "suricata:" >> $PILLARFILE
 	echo "  enabled: True " >> $PILLARFILE
+	if [[ $is_pcaplimit ]]; then
+	echo "  pcap:" >> $PILLARFILE
+	echo "    maxsize: $MAX_PCAP_SPACE" >> $PILLARFILE
+	fi
 	echo "  config:" >> $PILLARFILE
 	echo "    af-packet:" >> $PILLARFILE
 	echo "      threads: '$CORECOUNT'" >> $PILLARFILE
@@ -251,17 +281,11 @@ function add_sensor_to_minion() {
 	echo "  enabled: True" >> $PILLARFILE
 	if [[ $is_pcaplimit ]]; then
 		echo "  config:" >> $PILLARFILE
-		echo "    diskfreepercentage: 60" >> $PILLARFILE
+		echo "    diskfreepercentage: $DFREEPERCENT" >> $PILLARFILE
 	fi
 	echo "  " >> $PILLARFILE
 }
 
-function add_playbook_to_minion() {
-    printf '%s\n'\
-    "playbook:"\
-    "  enabled: True"\
-	"  " >> $PILLARFILE
-}
 
 function add_elastalert_to_minion() {
     printf '%s\n'\
@@ -323,13 +347,6 @@ function add_nginx_to_minion() {
 	"  " >> $PILLARFILE
 }
 
-function add_soctopus_to_minion() {
-    printf '%s\n'\
-    "soctopus:"\
-    "  enabled: True"\
-	"  " >> $PILLARFILE
-}
-
 function add_soc_to_minion() {
     printf '%s\n'\
     "soc:"\
@@ -344,13 +361,6 @@ function add_registry_to_minion() {
 	"  " >> $PILLARFILE
 }
 
-function add_mysql_to_minion() {
-    printf '%s\n'\
-    "mysql:"\
-    "  enabled: True"\
-	"  " >> $PILLARFILE
-}
-
 function add_kratos_to_minion() {
     printf '%s\n'\
     "kratos:"\
@@ -422,19 +432,17 @@ function updateMine() {
 
 function createEVAL() {
 	is_pcaplimit=true
+	pcapspace
 	add_elasticsearch_to_minion
 	add_sensor_to_minion
 	add_strelka_to_minion
-	add_playbook_to_minion
 	add_elastalert_to_minion
 	add_kibana_to_minion
 	add_telegraf_to_minion
 	add_influxdb_to_minion
 	add_nginx_to_minion
-	add_soctopus_to_minion
 	add_soc_to_minion
 	add_registry_to_minion
-	add_mysql_to_minion
 	add_kratos_to_minion
 	add_idstools_to_minion
 	add_elastic_fleet_package_registry_to_minion
@@ -442,21 +450,19 @@ function createEVAL() {
 
 function createSTANDALONE() {
 	is_pcaplimit=true
+	pcapspace
 	add_elasticsearch_to_minion
 	add_logstash_to_minion
 	add_sensor_to_minion
 	add_strelka_to_minion
-	add_playbook_to_minion
 	add_elastalert_to_minion
 	add_kibana_to_minion
 	add_redis_to_minion
 	add_telegraf_to_minion
 	add_influxdb_to_minion
 	add_nginx_to_minion
-	add_soctopus_to_minion
 	add_soc_to_minion
 	add_registry_to_minion
-	add_mysql_to_minion
 	add_kratos_to_minion
 	add_idstools_to_minion
 	add_elastic_fleet_package_registry_to_minion
@@ -465,17 +471,14 @@ function createSTANDALONE() {
 function createMANAGER() {
 	add_elasticsearch_to_minion
 	add_logstash_to_minion
-	add_playbook_to_minion
 	add_elastalert_to_minion
 	add_kibana_to_minion
 	add_redis_to_minion
 	add_telegraf_to_minion
 	add_influxdb_to_minion
 	add_nginx_to_minion
-	add_soctopus_to_minion
 	add_soc_to_minion
 	add_registry_to_minion
-	add_mysql_to_minion
 	add_kratos_to_minion
 	add_idstools_to_minion
 	add_elastic_fleet_package_registry_to_minion
@@ -484,17 +487,14 @@ function createMANAGER() {
 function createMANAGERSEARCH() {
 	add_elasticsearch_to_minion
 	add_logstash_to_minion
-	add_playbook_to_minion
 	add_elastalert_to_minion
 	add_kibana_to_minion
 	add_redis_to_minion
 	add_telegraf_to_minion
 	add_influxdb_to_minion
 	add_nginx_to_minion
-	add_soctopus_to_minion
 	add_soc_to_minion
 	add_registry_to_minion
-	add_mysql_to_minion
 	add_kratos_to_minion
 	add_idstools_to_minion
 	add_elastic_fleet_package_registry_to_minion
@@ -531,6 +531,9 @@ function createIDH() {
 
 function createHEAVYNODE() {
 	is_pcaplimit=true
+	PCAP_PERCENTAGE=1
+	DFREEPERCENT=21
+	pcapspace
 	add_elasticsearch_to_minion
 	add_elastic_agent_to_minion
 	add_logstash_to_minion
@@ -541,6 +544,10 @@ function createHEAVYNODE() {
 }
 
 function createSENSOR() {
+	is_pcaplimit=true
+	DFREEPERCENT=10
+	PCAP_PERCENTAGE=3
+	pcapspace
 	add_sensor_to_minion
 	add_strelka_to_minion
 	add_telegraf_to_minion

salt/manager/tools/sbin/so-repo-sync

@@ -7,12 +7,8 @@
 NOROOT=1
 . /usr/sbin/so-common
 
-set_version
-set_os
-salt_minion_count
-
 set -e
 
-curl --retry 5 --retry-delay 60 -A "reposync/$VERSION/$OS/$(uname -r)/$MINIONCOUNT" https://sigs.securityonion.net/checkup --output /tmp/checkup
+curl --retry 5 --retry-delay 60 -A "reposync/$(sync_options)" https://sigs.securityonion.net/checkup --output /tmp/checkup
 dnf reposync --norepopath -g --delete -m -c /opt/so/conf/reposync/repodownload.conf --repoid=securityonionsync --download-metadata -p /nsm/repo/
 createrepo /nsm/repo

salt/manager/tools/sbin/so-saltstack-update

@@ -47,7 +47,7 @@ got_root(){
 
 got_root
 if [ $# -ne 1 ] ; then
-  BRANCH=master
+  BRANCH=2.4/main
 else
   BRANCH=$1
 fi

salt/manager/tools/sbin/so-user

@@ -347,7 +347,7 @@ function syncElastic() {
     [[ $? != 0 ]] && fail "Unable to read credential hashes from database"
 
     user_data_formatted=$(echo "${userData}" | jq -r '.user + ":" + .data.hashed_password')
-    if lookup_salt_value "licensed_features" "" "pillar" | grep -x oidc; then
+    if lookup_salt_value "features" "" "pillar" | grep -x odc; then
       # generate random placeholder salt/hash for users without passwords
       random_crypt=$(get_random_value 53)
       user_data_formatted=$(echo "${user_data_formatted}" | sed -r "s/^(.+:)\$/\\1\$2a\$12${random_crypt}/")

salt/manager/tools/sbin/so-yaml.py

@@ -16,12 +16,14 @@ lockFile = "/tmp/so-yaml.lock"
 def showUsage(args):
     print('Usage: {} <COMMAND> <YAML_FILE> [ARGS...]'.format(sys.argv[0]))
     print('  General commands:')
+    print('    append         - Append a list item to a yaml key, if it exists and is a list. Requires KEY and LISTITEM args.')
     print('    remove         - Removes a yaml key, if it exists. Requires KEY arg.')
     print('    help           - Prints this usage information.')
     print('')
     print('  Where:')
     print('   YAML_FILE       - Path to the file that will be modified. Ex: /opt/so/conf/service/conf.yaml')
     print('   KEY             - YAML key, does not support \' or " characters at this time. Ex: level1.level2')
+    print('   LISTITEM        - Item to add to the list.')
     sys.exit(1)
 
 
@@ -35,6 +37,35 @@ def writeYaml(filename, content):
     file = open(filename, "w")
     return yaml.dump(content, file)
 
+def appendItem(content, key, listItem):
+    pieces = key.split(".", 1)
+    if len(pieces) > 1:
+        appendItem(content[pieces[0]], pieces[1], listItem)
+    else:
+        try:
+            content[key].append(listItem)
+        except AttributeError:
+            print("The existing value for the given key is not a list. No action was taken on the file.")
+            return 1
+        except KeyError:
+            print("The key provided does not exist. No action was taken on the file.")
+            return 1
+
+def append(args):
+    if len(args) != 3:
+        print('Missing filename, key arg, or list item to append', file=sys.stderr)
+        showUsage(None)
+        return
+
+    filename = args[0]
+    key = args[1]
+    listItem = args[2]
+
+    content = loadYaml(filename)
+    appendItem(content, key, listItem)
+    writeYaml(filename, content)
+
+    return 0
 
 def removeKey(content, key):
     pieces = key.split(".", 1)
@@ -69,6 +100,7 @@ def main():
 
     commands = {
         "help": showUsage,
+        "append": append,
         "remove": remove,
     }
 

salt/manager/tools/sbin/so-yaml_test.py

@@ -105,3 +105,99 @@ class TestRemove(unittest.TestCase):
                 self.assertEqual(actual, expected)
                 sysmock.assert_called_once_with(1)
                 self.assertIn(mock_stdout.getvalue(), "Missing filename or key arg\n")
+
+    def test_append(self):
+        filename = "/tmp/so-yaml_test-remove.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: abc }, key2: false, key3: [a,b,c]}")
+        file.close()
+
+        soyaml.append([filename, "key3", "d"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+        expected = "key1:\n  child1: 123\n  child2: abc\nkey2: false\nkey3:\n- a\n- b\n- c\n- d\n"
+        self.assertEqual(actual, expected)
+
+    def test_append_nested(self):
+        filename = "/tmp/so-yaml_test-remove.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: [a,b,c] }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        soyaml.append([filename, "key1.child2", "d"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2:\n  - a\n  - b\n  - c\n  - d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
+        self.assertEqual(actual, expected)
+
+    def test_append_nested_deep(self):
+        filename = "/tmp/so-yaml_test-remove.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        soyaml.append([filename, "key1.child2.deep2", "d"])
+
+        file = open(filename, "r")
+        actual = file.read()
+        file.close()
+
+        expected = "key1:\n  child1: 123\n  child2:\n    deep1: 45\n    deep2:\n    - a\n    - b\n    - c\n    - d\nkey2: false\nkey3:\n- e\n- f\n- g\n"
+        self.assertEqual(actual, expected)
+
+    def test_append_key_noexist(self):
+        filename = "/tmp/so-yaml_test-append.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+                sys.argv = ["cmd", "append", filename, "key4", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n")
+
+    def test_append_key_noexist_deep(self):
+        filename = "/tmp/so-yaml_test-append.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+                sys.argv = ["cmd", "append", filename, "key1.child2.deep3", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual(mock_stdout.getvalue(), "The key provided does not exist. No action was taken on the file.\n")
+
+    def test_append_key_nonlist(self):
+        filename = "/tmp/so-yaml_test-append.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+                sys.argv = ["cmd", "append", filename, "key1", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")
+
+    def test_append_key_nonlist_deep(self):
+        filename = "/tmp/so-yaml_test-append.yaml"
+        file = open(filename, "w")
+        file.write("{key1: { child1: 123, child2: { deep1: 45, deep2: [a,b,c] } }, key2: false, key3: [e,f,g]}")
+        file.close()
+
+        with patch('sys.exit', new=MagicMock()) as sysmock:
+            with patch('sys.stdout', new=StringIO()) as mock_stdout:
+                sys.argv = ["cmd", "append", filename, "key1.child2.deep1", "h"]
+                soyaml.main()
+                sysmock.assert_called()
+                self.assertEqual(mock_stdout.getvalue(), "The existing value for the given key is not a list. No action was taken on the file.\n")

salt/manager/tools/sbin/soup

@@ -247,67 +247,6 @@ check_sudoers() {
   fi
 }
 
-check_log_size_limit() {
-  local num_minion_pillars
-  num_minion_pillars=$(find /opt/so/saltstack/local/pillar/minions/ -type f | wc -l)
-  
-  if [[ $num_minion_pillars -gt 1 ]]; then
-    if find /opt/so/saltstack/local/pillar/minions/ -type f | grep -q "_heavynode"; then
-      lsl_msg='distributed'
-    fi
-  else
-    local minion_id
-    minion_id=$(lookup_salt_value "id" "" "grains" "" "local")
-
-    local minion_arr
-    IFS='_' read -ra minion_arr <<< "$minion_id"
-
-    local node_type="${minion_arr[0]}"
-    
-    local current_limit
-    # since it is possible for the salt-master service to be stopped when this is run, we need to check the pillar values locally
-    # we need to combine default local and default pillars before doing this so we can define --pillar-root in salt-call
-    local epoch_date=$(date +%s%N)
-    mkdir -vp /opt/so/saltstack/soup_tmp_${epoch_date}/
-    cp -r /opt/so/saltstack/default/pillar/ /opt/so/saltstack/soup_tmp_${epoch_date}/
-    # use \cp here to overwrite any pillar files from default with those in local for the tmp directory
-    \cp -r /opt/so/saltstack/local/pillar/ /opt/so/saltstack/soup_tmp_${epoch_date}/
-    current_limit=$(salt-call pillar.get elasticsearch:log_size_limit --local --pillar-root=/opt/so/saltstack/soup_tmp_${epoch_date}/pillar --out=newline_values_only)
-    rm -rf /opt/so/saltstack/soup_tmp_${epoch_date}/
-    
-    local percent
-    case $node_type in
-      'standalone' | 'eval')
-        percent=50
-      ;;
-      *)
-        percent=80
-      ;;
-    esac
-
-    local disk_dir="/"
-    if [ -d /nsm ]; then
-      disk_dir="/nsm"
-    fi
-
-    local disk_size_1k
-    disk_size_1k=$(df $disk_dir | grep -v "^Filesystem" | awk '{print $2}')
-
-    local ratio="1048576"
-
-    local disk_size_gb
-    disk_size_gb=$( echo "$disk_size_1k" "$ratio" | awk '{print($1/$2)}' )
-
-    local new_limit
-    new_limit=$( echo "$disk_size_gb" "$percent" | awk '{printf("%.0f", $1 * ($2/100))}')
-
-    if [[ $current_limit != "$new_limit" ]]; then
-      lsl_msg='single-node'
-      lsl_details=( "$current_limit" "$new_limit" "$minion_id" )
-    fi
-  fi
-}
-
 check_os_updates() {
   # Check to see if there are OS updates
   echo "Checking for OS updates."
@@ -372,6 +311,17 @@ enable_highstate() {
     echo ""
 }
 
+get_soup_script_hashes() {
+  CURRENTSOUP=$(md5sum /usr/sbin/soup | awk '{print $1}')
+  GITSOUP=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/soup | awk '{print $1}')
+  CURRENTCMN=$(md5sum /usr/sbin/so-common | awk '{print $1}')
+  GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}')
+  CURRENTIMGCMN=$(md5sum /usr/sbin/so-image-common | awk '{print $1}')
+  GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
+  CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}')
+  GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-firewall | awk '{print $1}')
+}
+
 highstate() {
   # Run a highstate.
   salt-call state.highstate -l info queue=True
@@ -405,6 +355,8 @@ preupgrade_changes() {
     [[ "$INSTALLEDVERSION" == 2.4.10 ]] && up_to_2.4.20
     [[ "$INSTALLEDVERSION" == 2.4.20 ]] && up_to_2.4.30
     [[ "$INSTALLEDVERSION" == 2.4.30 ]] && up_to_2.4.40
+    [[ "$INSTALLEDVERSION" == 2.4.40 ]] && up_to_2.4.50
+    [[ "$INSTALLEDVERSION" == 2.4.50 ]] && up_to_2.4.60
     true
 }
 
@@ -419,6 +371,8 @@ postupgrade_changes() {
     [[ "$POSTVERSION" == 2.4.10 ]] && post_to_2.4.20
     [[ "$POSTVERSION" == 2.4.20 ]] && post_to_2.4.30
     [[ "$POSTVERSION" == 2.4.30 ]] && post_to_2.4.40
+    [[ "$POSTVERSION"  == 2.4.40 ]] && post_to_2.4.50
+    [[ "$POSTVERSION"  == 2.4.50 ]] && post_to_2.4.60
     true
 }
 
@@ -470,6 +424,17 @@ post_to_2.4.40() {
   POSTVERSION=2.4.40
 }
 
+post_to_2.4.50() {
+  echo "Nothing to apply"
+  POSTVERSION=2.4.50
+}
+
+post_to_2.4.60() {
+  echo "Regenerating Elastic Agent Installers..."
+  so-elastic-agent-gen-installers
+  POSTVERSION=2.4.60
+}
+
 repo_sync() {
   echo "Sync the local repo."
   su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync."
@@ -570,6 +535,45 @@ up_to_2.4.40() {
   INSTALLEDVERSION=2.4.40
 }
 
+up_to_2.4.50() {
+  echo "Creating additional pillars.."
+  mkdir -p /opt/so/saltstack/local/pillar/stig/
+  mkdir -p /opt/so/saltstack/local/salt/stig/
+  chown socore:socore /opt/so/saltstack/local/salt/stig/
+  touch /opt/so/saltstack/local/pillar/stig/adv_stig.sls
+  touch /opt/so/saltstack/local/pillar/stig/soc_stig.sls
+
+  # the file_roots need to be update due to salt 3006.6 upgrade not allowing symlinks outside the file_roots
+  # put new so-yaml in place
+  echo "Updating so-yaml"
+  \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" "$DEFAULT_SALT_DIR/salt/manager/tools/sbin/"
+  \cp -v "$UPDATE_DIR/salt/manager/tools/sbin/so-yaml.py" /usr/sbin/
+  echo "Creating a backup of the salt-master config."
+  # INSTALLEDVERSION is 2.4.40 at this point, but we want the backup to have the version
+  # so was at prior to starting upgrade. use POSTVERSION here since it doesnt change until
+  # post upgrade changes. POSTVERSION set to INSTALLEDVERSION at start of soup
+  cp -v /etc/salt/master "/etc/salt/master.so-$POSTVERSION.bak"
+  echo "Adding /opt/so/rules to file_roots in /etc/salt/master using so-yaml"
+  so-yaml.py append /etc/salt/master file_roots.base /opt/so/rules/nids
+  echo "Moving Suricata rules"
+  mkdir /opt/so/rules/nids/suri
+  chown socore:socore /opt/so/rules/nids/suri
+  mv -v /opt/so/rules/nids/*.rules /opt/so/rules/nids/suri/.
+  
+  echo "Adding /nsm/elastic-fleet/artifacts to file_roots in /etc/salt/master using so-yaml"
+  so-yaml.py append /etc/salt/master file_roots.base  /nsm/elastic-fleet/artifacts
+
+  INSTALLEDVERSION=2.4.50
+}
+
+up_to_2.4.60() {
+  echo "Creating directory to store Suricata classification.config"
+  mkdir -vp /opt/so/saltstack/local/salt/suricata/classification
+  chown socore:socore /opt/so/saltstack/local/salt/suricata/classification
+
+  INSTALLEDVERSION=2.4.60
+}
+
 determine_elastic_agent_upgrade() {
   if [[ $is_airgap -eq 0 ]]; then
     update_elastic_agent_airgap
@@ -617,6 +621,10 @@ update_airgap_rules() {
   if [ -d /nsm/repo/rules/sigma ]; then
     rsync -av $UPDATE_DIR/agrules/sigma/* /nsm/repo/rules/sigma/
   fi
+
+  # SOC Detections Airgap
+  rsync -av $UPDATE_DIR/agrules/detect-sigma/* /nsm/rules/detect-sigma/
+  rsync -av $UPDATE_DIR/agrules/detect-yara/* /nsm/rules/detect-yara/
 }
 
 update_airgap_repo() {
@@ -742,31 +750,29 @@ upgrade_salt() {
 }
 
 verify_latest_update_script() {
-  # Check to see if the update scripts match. If not run the new one.
-  CURRENTSOUP=$(md5sum /usr/sbin/soup | awk '{print $1}')
-  GITSOUP=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/soup | awk '{print $1}')
-  CURRENTCMN=$(md5sum /usr/sbin/so-common | awk '{print $1}')
-  GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}')
-  CURRENTIMGCMN=$(md5sum /usr/sbin/so-image-common | awk '{print $1}')
-  GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
-  CURRENTSOFIREWALL=$(md5sum /usr/sbin/so-firewall | awk '{print $1}')
-  GITSOFIREWALL=$(md5sum $UPDATE_DIR/salt/manager/tools/sbin/so-firewall | awk '{print $1}')
-
+  get_soup_script_hashes
   if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
     echo "This version of the soup script is up to date. Proceeding."
   else
     echo "You are not running the latest soup version. Updating soup and its components. This might take multiple runs to complete."
-    cp $UPDATE_DIR/salt/manager/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/
-    cp $UPDATE_DIR/salt/common/tools/sbin/so-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
-    cp $UPDATE_DIR/salt/common/tools/sbin/so-image-common $DEFAULT_SALT_DIR/salt/common/tools/sbin/
-    cp $UPDATE_DIR/salt/manager/tools/sbin/so-firewall $DEFAULT_SALT_DIR/salt/common/tools/sbin/
+
     salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local
+
+    # Verify that soup scripts updated as expected
+    get_soup_script_hashes
+    if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" && "$CURRENTSOFIREWALL" == "$GITSOFIREWALL" ]]; then
+      echo "Succesfully updated soup scripts."
+    else
+      echo "There was a problem updating soup scripts. Trying to rerun script update."
+      salt-call state.apply common.soup_scripts queue=True -linfo --file-root=$UPDATE_DIR/salt --local
+    fi
+
     echo ""
     echo "The soup script has been modified. Please run soup again to continue the upgrade."
     exit 0
   fi
-}
 
+}
 # Keeping this block in case we need to do a hotfix that requires salt update
 apply_hotfix() {
   if [[ "$INSTALLEDVERSION" == "2.4.20" ]] ; then
@@ -909,9 +915,6 @@ main() {
 
     systemctl_func "stop" "$cron_service_name"
 
-    # update mine items prior to stopping salt-minion and salt-master
-    update_salt_mine
-
     echo "Updating dockers to $NEWVERSION."
     if [[ $is_airgap -eq 0 ]]; then
       airgap_update_dockers
@@ -987,6 +990,9 @@ main() {
     salt-call state.apply salt.minion -l info queue=True
     echo ""
 
+    # ensure the mine is updated and populated before highstates run, following the salt-master restart
+    update_salt_mine
+
     enable_highstate
 
     echo ""

salt/mysql/config.sls

@@ -1,89 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-{%   set MYSQLPASS = salt['pillar.get']('secrets:mysql') %}
-
-# MySQL Setup
-mysqlpkgs:
-  pkg.removed:
-    - skip_suggestions: False
-    - pkgs:
-      {% if grains['os_family'] != 'RedHat' %}
-      - python3-mysqldb
-      {% else %}
-      - python3-mysqlclient
-      {% endif %}
-
-mysqletcdir:
-  file.directory:
-    - name: /opt/so/conf/mysql/etc
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-mysqlpiddir:
-  file.directory:
-    - name: /opt/so/conf/mysql/pid
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-mysqlcnf:
-  file.managed:
-    - name: /opt/so/conf/mysql/etc/my.cnf
-    - source: salt://mysql/etc/my.cnf
-    - user: 939
-    - group: 939
-
-mysqlpass:
-  file.managed:
-    - name: /opt/so/conf/mysql/etc/mypass
-    - source: salt://mysql/etc/mypass
-    - user: 939
-    - group: 939
-    - template: jinja
-    - defaults:
-        MYSQLPASS: {{ MYSQLPASS }}
-
-mysqllogdir:
-  file.directory:
-    - name: /opt/so/log/mysql
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-mysqldatadir:
-  file.directory:
-    - name: /nsm/mysql
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-mysql_sbin:
-  file.recurse:
-    - name: /usr/sbin
-    - source: salt://mysql/tools/sbin
-    - user: 939
-    - group: 939
-    - file_mode: 755
-
-#mysql_sbin_jinja:
-#  file.recurse:
-#    - name: /usr/sbin
-#    - source: salt://mysql/tools/sbin_jinja
-#    - user: 939
-#    - group: 939 
-#    - file_mode: 755
-#    - template: jinja
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/mysql/defaults.yaml

@@ -1,2 +0,0 @@
-mysql:
-  enabled: False

salt/mysql/disabled.sls

@@ -1,27 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-
-include:
-  - mysql.sostatus
-  
-so-mysql:
-  docker_container.absent:
-    - force: True
-
-so-mysql_so-status.disabled:
-  file.comment:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - regex: ^so-mysql$
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/mysql/enabled.sls

@@ -1,84 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-{%   from 'docker/docker.map.jinja' import DOCKER %}
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   set MYSQLPASS = salt['pillar.get']('secrets:mysql') %}
-
-include:
-  - mysql.config
-  - mysql.sostatus
-
-{% if MYSQLPASS == None %}
-
-mysql_password_none:
-  test.configurable_test_state:
-    - changes: False
-    - result: False
-    - comment: "MySQL Password Error - Not Starting MySQL"
-
-{% else %}
-
-so-mysql:
-  docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-mysql:{{ GLOBALS.so_version }}
-    - hostname: so-mysql
-    - user: socore
-    - networks:
-      - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-mysql'].ip }}
-    - extra_hosts:
-      - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
-      {% if DOCKER.containers['so-mysql'].extra_hosts %}
-        {% for XTRAHOST in DOCKER.containers['so-mysql'].extra_hosts %}
-      - {{ XTRAHOST }}
-        {% endfor %}
-      {% endif %}
-    - port_bindings:
-      {% for BINDING in DOCKER.containers['so-mysql'].port_bindings %}
-      - {{ BINDING }}
-      {% endfor %}
-    - environment:
-      - MYSQL_ROOT_HOST={{ GLOBALS.so_docker_gateway }}
-      - MYSQL_ROOT_PASSWORD=/etc/mypass
-      {% if DOCKER.containers['so-mysql'].extra_env %}
-        {% for XTRAENV in DOCKER.containers['so-mysql'].extra_env %}
-        - {{ XTRAENV }}
-      {% endfor %}
-    {% endif %}
-    - binds:
-      - /opt/so/conf/mysql/etc/my.cnf:/etc/my.cnf:ro
-      - /opt/so/conf/mysql/etc/mypass:/etc/mypass
-      - /nsm/mysql:/var/lib/mysql:rw
-      - /opt/so/log/mysql:/var/log/mysql:rw
-      {% if DOCKER.containers['so-mysql'].custom_bind_mounts %}
-        {% for BIND in DOCKER.containers['so-mysql'].custom_bind_mounts %}
-      - {{ BIND }}
-        {% endfor %}
-      {% endif %}
-    - cap_add:
-      - SYS_NICE
-    - watch:
-      - file: mysqlcnf
-      - file: mysqlpass
-    - require:
-      - file: mysqlcnf
-      - file: mysqlpass
-{% endif %}
-
-delete_so-mysql_so-status.disabled:
-  file.uncomment:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - regex: ^so-mysql$
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/mysql/etc/my.cnf

@@ -1,32 +0,0 @@
-# For advice on how to change settings please see
-# http://dev.mysql.com/doc/refman/5.7/en/server-configuration-defaults.html
-
-[mysqld]
-#
-# Remove leading # and set to the amount of RAM for the most important data
-# cache in MySQL. Start at 70% of total RAM for dedicated server, else 10%.
-# innodb_buffer_pool_size = 128M
-#
-# Remove leading # to turn on a very important data integrity option: logging
-# changes to the binary log between backups.
-# log_bin
-#
-# Remove leading # to set options mainly useful for reporting servers.
-# The server defaults are faster for transactions and fast SELECTs.
-# Adjust sizes as needed, experiment to find the optimal values.
-# join_buffer_size = 128M
-# sort_buffer_size = 2M
-# read_rnd_buffer_size = 2M
-
-host_cache_size=0
-skip-name-resolve
-datadir=/var/lib/mysql
-socket=/var/lib/mysql/mysql.sock
-secure-file-priv=/var/lib/mysql-files
-user=socore
-
-log-error=/var/log/mysql/mysqld.log
-pid-file=/var/run/mysqld/mysqld.pid
-
-# Switch back to the native password module so that playbook can connect
-authentication_policy=mysql_native_password

salt/mysql/etc/mypass

@@ -1 +0,0 @@
-{{ MYSQLPASS }}

salt/mysql/init.sls

@@ -1,14 +0,0 @@
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'mysql/map.jinja' import MYSQLMERGED %}
-
-include:
-{% if MYSQLMERGED.enabled %}
-  - mysql.enabled
-{% else %}
-  - mysql.disabled
-{% endif %}

salt/mysql/soc_mysql.yaml

@@ -1,4 +0,0 @@
-mysql:
-  enabled:
-    description: You can enable or disable MySQL.
-    advanced: True

salt/mysql/tools/sbin/so-mysql-start

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-start mysql $1

salt/mysql/tools/sbin/so-mysql-stop

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-stop mysql $1

salt/nginx/enabled.sls

@@ -14,6 +14,9 @@ include:
   - nginx.config
   - nginx.sostatus
 
+
+{%   if grains.role not in ['so-fleet'] %}
+
 {#   if the user has selected to replace the crt and key in the ui #}
 {%   if NGINXMERGED.ssl.replace_cert %}
 
@@ -89,17 +92,26 @@ make-rule-dir-nginx:
       - user
       - group
       
+{%   endif %}
+
+{# if this is an so-fleet node then we want to use the port bindings, custom bind mounts defined for fleet #}
+{% if GLOBALS.role == 'so-fleet' %}
+{%   set container_config = 'so-nginx-fleet-node' %}
+{% else %}
+{%   set container_config = 'so-nginx' %}
+{% endif %}
+
 so-nginx:
   docker_container.running:
     - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-nginx:{{ GLOBALS.so_version }}
     - hostname: so-nginx
     - networks:
       - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-nginx'].ip }}
+        - ipv4_address: {{ DOCKER.containers[container_config].ip }}
     - extra_hosts:
       - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
-    {% if DOCKER.containers['so-nginx'].extra_hosts %}
-      {% for XTRAHOST in DOCKER.containers['so-nginx'].extra_hosts %}
+    {% if DOCKER.containers[container_config].extra_hosts %}
+      {% for XTRAHOST in DOCKER.containers[container_config].extra_hosts %}
       - {{ XTRAHOST }}
       {% endfor %}
     {% endif %}
@@ -119,20 +131,20 @@ so-nginx:
       - /nsm/repo:/opt/socore/html/repo:ro
       - /nsm/rules:/nsm/rules:ro
       {% endif %}
-      {% if DOCKER.containers['so-nginx'].custom_bind_mounts %}
-        {% for BIND in DOCKER.containers['so-nginx'].custom_bind_mounts %}
+      {% if DOCKER.containers[container_config].custom_bind_mounts %}
+        {% for BIND in DOCKER.containers[container_config].custom_bind_mounts %}
       - {{ BIND }}
         {% endfor %}
       {% endif %}
-    {% if DOCKER.containers['so-nginx'].extra_env %}
+    {% if DOCKER.containers[container_config].extra_env %}
     - environment:
-      {% for XTRAENV in DOCKER.containers['so-nginx'].extra_env %}
+      {% for XTRAENV in DOCKER.containers[container_config].extra_env %}
       - {{ XTRAENV }}
       {% endfor %}
     {% endif %}
     - cap_add: NET_BIND_SERVICE
     - port_bindings:
-      {% for BINDING in DOCKER.containers['so-nginx'].port_bindings %}
+      {% for BINDING in DOCKER.containers[container_config].port_bindings %}
       - {{ BINDING }}
       {% endfor %}
     - watch:

salt/nginx/etc/nginx.conf

@@ -39,6 +39,26 @@ http {
 
 	include /etc/nginx/conf.d/*.conf;
 
+	{%- if role in ['fleet'] %}
+
+	server {
+		listen 8443;
+		server_name {{ GLOBALS.hostname }};
+		root /opt/socore/html;
+		location /artifacts/ {
+			try_files $uri =206;
+			proxy_read_timeout    90;
+			proxy_connect_timeout 90;
+			proxy_set_header      Host $host;
+			proxy_set_header      X-Real-IP $remote_addr;
+			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
+			proxy_set_header      Proxy "";
+			proxy_set_header      X-Forwarded-Proto $scheme;
+		}
+	}
+
+    {%- endif %}
+
 	{%- if role in ['eval', 'managersearch', 'manager', 'standalone', 'import'] %}
 
 	server {
@@ -257,38 +277,11 @@ http {
 			proxy_set_header      X-Forwarded-Proto $scheme;
 		}
 
-		location /playbook/ {
-			auth_request          /auth/sessions/whoami;
-			proxy_pass            http://{{ GLOBALS.manager }}:3000/playbook/;
-			proxy_read_timeout    90;
-			proxy_connect_timeout 90;
-			proxy_set_header      Host $host;
-			proxy_set_header      X-Real-IP $remote_addr;
-			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
-			proxy_set_header      Proxy "";
-			proxy_set_header      X-Forwarded-Proto $scheme;
-		}
-
-
-		location /soctopus/ {
-			auth_request          /auth/sessions/whoami;
-			proxy_pass            http://{{ GLOBALS.manager }}:7000/;
-			proxy_read_timeout    300;
-			proxy_connect_timeout 300;
-			proxy_set_header      Host $host;
-			proxy_set_header      X-Real-IP $remote_addr;
-			proxy_set_header      X-Forwarded-For $proxy_add_x_forwarded_for;
-			proxy_set_header      Proxy "";
-			proxy_set_header      X-Forwarded-Proto $scheme;
-		}
 
 		location /kibana/app/soc/ {
 			rewrite ^/kibana/app/soc/(.*) /soc/$1 permanent;
 		}
 
-		location /kibana/app/soctopus/ {
-			rewrite ^/kibana/app/soctopus/(.*) /soctopus/$1 permanent;
-		}
 
 		location /sensoroniagents/ {
 			if ($http_authorization = "") {

salt/pcap/config.map.jinja

@@ -3,5 +3,11 @@
    https://securityonion.net/license; you may not use this file except in compliance with the
    Elastic License 2.0. #}
 
+{% from 'vars/globals.map.jinja' import GLOBALS %}
 {% import_yaml 'pcap/defaults.yaml' as PCAPDEFAULTS %}
 {% set PCAPMERGED = salt['pillar.get']('pcap', PCAPDEFAULTS.pcap, merge=True) %}
+
+{# disable stenographer if the pcap engine is set to SURICATA #}
+{% if GLOBALS.pcap_engine == "SURICATA" %}
+{%   do PCAPMERGED.update({'enabled': False}) %}
+{% endif %}

salt/pcap/config.sls

@@ -72,13 +72,6 @@ stenoca:
     - user: 941
     - group: 939
 
-pcapdir:
-  file.directory:
-    - name: /nsm/pcap
-    - user: 941
-    - group: 941
-    - makedirs: True
-
 pcaptmpdir:
   file.directory:
     - name: /nsm/pcaptmp

salt/pcap/init.sls

@@ -15,3 +15,12 @@ include:
 {% else %}
   - pcap.disabled
 {% endif %}
+
+# This directory needs to exist regardless of whether STENO is enabled or not, in order for
+# Sensoroni to be able to look at old steno PCAP data
+pcapdir:
+  file.directory:
+    - name: /nsm/pcap
+    - user: 941
+    - group: 941
+    - makedirs: True

salt/pcap/soc_pcap.yaml

@@ -4,32 +4,32 @@ pcap:
     helpLink: stenographer.html
   config:
     maxdirectoryfiles:
-      description: The maximum number of packet/index files to create before deleting old files.
+      description: By default, Stenographer limits the number of files in the pcap directory to 30000 to avoid limitations with the ext3 filesystem. However, if you're using the ext4 or xfs filesystems, then it is safe to increase this value. So if you have a large amount of storage and find that you only have 3 weeks worth of PCAP on disk while still having plenty of free space, then you may want to increase this default setting.
       helpLink: stenographer.html
     diskfreepercentage:
-      description: The disk space percent to always keep free for PCAP
+      description: Stenographer will purge old PCAP on a regular basis to keep the disk free percentage at this level. If you have a distributed deployment with dedicated forward nodes, then the default value of 10 should be reasonable since Stenographer should be the main consumer of disk space in the /nsm partition. However, if you have systems that run both Stenographer and Elasticsearch at the same time (like eval and standalone installations), then you’ll want to make sure that this value is no lower than 21 so that you avoid Elasticsearch hitting its watermark setting at 80% disk usage. If you have an older standalone installation, then you may need to manually change this value to 21.
       helpLink: stenographer.html
     blocks:
-      description: The number of 1MB packet blocks used by AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. 
+      description: The number of 1MB packet blocks used by Stenographer and AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. 
       advanced: True
       helpLink: stenographer.html
     preallocate_file_mb:
-      description: File size to pre-allocate for individual PCAP files. You shouldn't need to change this. 
+      description: File size to pre-allocate for individual Stenographer PCAP files. You shouldn't need to change this. 
       advanced: True
       helpLink: stenographer.html
     aiops:
-      description: The max number of async writes to allow at once.
+      description: The max number of async writes to allow for Stenographer at once.
       advanced: True
       helpLink: stenographer.html
     pin_to_cpu:
-      description: Enable CPU pinning for PCAP.
+      description: Enable CPU pinning for Stenographer PCAP.
       advanced: True
       helpLink: stenographer.html
     cpus_to_pin_to:
-      description: CPU to pin PCAP to. Currently only a single CPU is supported.
+      description: CPU to pin Stenographer PCAP to. Currently only a single CPU is supported.
       advanced: True
       helpLink: stenographer.html
     disks:
-      description: List of disks to use for PCAP. This is currently not used.
+      description: List of disks to use for Stenographer PCAP. This is currently not used.
       advanced: True
       helpLink: stenographer.html

salt/playbook/automation_user_create.sls

@@ -1,19 +0,0 @@
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
-# This state will create the SecOps Automation user within Playbook
-
-include:
-  - playbook
-  
-wait_for_playbook:
-  cmd.run:
-    - name: until nc -z {{ GLOBALS.manager }} 3000; do sleep 1; done
-    - timeout: 300
-
-create_user:
-  cmd.script:
-    - source: salt://playbook/files/automation_user_create.sh
-    - cwd: /root
-    - template: jinja
-    - onchanges:
-      - cmd: wait_for_playbook

salt/playbook/config.sls

@@ -1,120 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-{%   from 'docker/docker.map.jinja' import DOCKER %}
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   set MYSQLPASS = salt['pillar.get']('secrets:mysql') %}
-{%   set PLAYBOOKPASS = salt['pillar.get']('secrets:playbook_db') %}
-
-
-include:
-  - mysql
-  
-create_playbookdbuser:
-  mysql_user.present:
-    - name: playbookdbuser
-    - password: {{ PLAYBOOKPASS }}
-    - host: "{{ DOCKER.range.split('/')[0] }}/255.255.255.0"
-    - connection_host: {{ GLOBALS.manager }}
-    - connection_port: 3306
-    - connection_user: root
-    - connection_pass: {{ MYSQLPASS }}
-
-query_playbookdbuser_grants:
-  mysql_query.run:
-    - database: playbook
-    - query:    "GRANT ALL ON playbook.* TO 'playbookdbuser'@'{{ DOCKER.range.split('/')[0] }}/255.255.255.0';"
-    - connection_host: {{ GLOBALS.manager }}
-    - connection_port: 3306
-    - connection_user: root
-    - connection_pass: {{ MYSQLPASS }}
-
-query_updatwebhooks:
-  mysql_query.run:
-    - database: playbook
-    - query:    "update webhooks set url = 'http://{{ GLOBALS.manager_ip}}:7000/playbook/webhook' where project_id = 1"
-    - connection_host: {{ GLOBALS.manager }}
-    - connection_port: 3306
-    - connection_user: root
-    - connection_pass: {{ MYSQLPASS }}
-
-query_updatename:
-  mysql_query.run:
-    - database: playbook
-    - query:    "update custom_fields set name = 'Custom Filter' where id = 21;"
-    - connection_host: {{ GLOBALS.manager }}
-    - connection_port: 3306
-    - connection_user: root
-    - connection_pass: {{ MYSQLPASS }} 
-
-query_updatepluginurls:
-  mysql_query.run:
-    - database: playbook
-    - query: |- 
-        update settings set value = 
-        "--- !ruby/hash:ActiveSupport::HashWithIndifferentAccess
-        project: '1'
-        convert_url: http://{{ GLOBALS.manager }}:7000/playbook/sigmac
-        create_url: http://{{ GLOBALS.manager }}:7000/playbook/play"
-        where id  = 43
-    - connection_host: {{ GLOBALS.manager }}
-    - connection_port: 3306
-    - connection_user: root
-    - connection_pass: {{ MYSQLPASS }}
-
-playbook_sbin:
-  file.recurse:
-    - name: /usr/sbin
-    - source: salt://playbook/tools/sbin
-    - user: 939
-    - group: 939
-    - file_mode: 755
-
-#playbook_sbin_jinja:
-#  file.recurse:
-#    - name: /usr/sbin
-#    - source: salt://playbook/tools/sbin_jinja
-#    - user: 939
-#    - group: 939 
-#    - file_mode: 755
-#    - template: jinja
-
-playbooklogdir:
-  file.directory:
-    - name: /opt/so/log/playbook
-    - dir_mode: 775
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-playbookfilesdir:
-  file.directory:
-    - name: /opt/so/conf/playbook/redmine-files
-    - dir_mode: 775
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-{%   if 'idh' in salt['cmd.shell']("ls /opt/so/saltstack/local/pillar/minions/|awk -F'_' {'print $2'}|awk -F'.' {'print $1'}").split() %}
-idh-plays:
-  file.recurse:
-    - name: /opt/so/conf/soctopus/sigma-import
-    - source: salt://idh/plays
-    - makedirs: True
-  cmd.run:
-    - name: so-playbook-import True
-    - onchanges:
-      - file: /opt/so/conf/soctopus/sigma-import
-{%   endif %}
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/playbook/db_init.sls

@@ -1,14 +0,0 @@
-
-# This state will import the initial default playbook database. 
-# If there is an existing playbook database, it will be overwritten - no backups are made.
-
-include:
-  - mysql
-
-salt://playbook/files/playbook_db_init.sh:
-  cmd.script:
-    - cwd: /root
-    - template: jinja
-
-'sleep 5':
-  cmd.run

salt/playbook/defaults.yaml

@@ -1,2 +0,0 @@
-playbook:
-  enabled: False

salt/playbook/disabled.sls

@@ -1,37 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-
-include:
-  - playbook.sostatus
-  
-so-playbook:
-  docker_container.absent:
-    - force: True
-
-so-playbook_so-status.disabled:
-  file.comment:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - regex: ^so-playbook$
-
-so-playbook-sync_cron:
-  cron.absent:
-    - identifier: so-playbook-sync_cron
-    - user: root
-
-so-playbook-ruleupdate_cron:
-  cron.absent:
-    - identifier: so-playbook-ruleupdate_cron
-    - user: root
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/playbook/enabled.sls

@@ -1,93 +0,0 @@
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls.split('.')[0] in allowed_states %}
-
-{%   from 'docker/docker.map.jinja' import DOCKER %}
-{%   from 'vars/globals.map.jinja' import GLOBALS %}
-{%   set PLAYBOOKPASS = salt['pillar.get']('secrets:playbook_db') %}
-
-include:
-  - playbook.config
-  - playbook.sostatus
-
-{%   if PLAYBOOKPASS == None %}
-
-playbook_password_none:
-  test.configurable_test_state:
-    - changes: False
-    - result: False
-    - comment: "Playbook MySQL Password Error - Not Starting Playbook"
-
-{%   else %}
-
-so-playbook:
-  docker_container.running:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-playbook:{{ GLOBALS.so_version }}
-    - hostname: playbook
-    - name: so-playbook
-    - networks:
-      - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-playbook'].ip }}
-    - binds:
-      - /opt/so/conf/playbook/redmine-files:/usr/src/redmine/files:rw
-      - /opt/so/log/playbook:/playbook/log:rw
-      {% if DOCKER.containers['so-playbook'].custom_bind_mounts %}
-        {% for BIND in DOCKER.containers['so-playbook'].custom_bind_mounts %}
-      - {{ BIND }}
-        {% endfor %}
-      {% endif %}
-    - extra_hosts:
-      - {{ GLOBALS.manager }}:{{ GLOBALS.manager_ip }}
-      {% if DOCKER.containers['so-playbook'].extra_hosts %}
-        {% for XTRAHOST in DOCKER.containers['so-playbook'].extra_hosts %}
-      - {{ XTRAHOST }}
-        {% endfor %}
-      {% endif %}
-    - environment:
-      - REDMINE_DB_MYSQL={{ GLOBALS.manager }}
-      - REDMINE_DB_DATABASE=playbook
-      - REDMINE_DB_USERNAME=playbookdbuser
-      - REDMINE_DB_PASSWORD={{ PLAYBOOKPASS }}
-      {% if DOCKER.containers['so-playbook'].extra_env %}
-        {% for XTRAENV in DOCKER.containers['so-playbook'].extra_env %}
-      - {{ XTRAENV }}
-        {% endfor %}
-      {% endif %}
-    - port_bindings:
-      {% for BINDING in DOCKER.containers['so-playbook'].port_bindings %}
-      - {{ BINDING }}
-      {% endfor %}
-
-delete_so-playbook_so-status.disabled:
-  file.uncomment:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - regex: ^so-playbook$
-
-so-playbook-sync_cron:
-  cron.present:
-    - name: /usr/sbin/so-playbook-sync > /opt/so/log/playbook/sync.log 2>&1
-    - identifier: so-playbook-sync_cron
-    - user: root
-    - minute: '*/5'
-
-so-playbook-ruleupdate_cron:
-  cron.present:
-    - name: /usr/sbin/so-playbook-ruleupdate > /opt/so/log/playbook/update.log 2>&1
-    - identifier: so-playbook-ruleupdate_cron
-    - user: root
-    - minute: '1'
-    - hour: '6'
-
-{%   endif %}
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}

salt/playbook/files/automation_user_create.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-# {%- set admin_pass = salt['pillar.get']('secrets:playbook_admin', None) -%}
-# {%- set automation_pass = salt['pillar.get']('secrets:playbook_automation', None) %}
-
-local_salt_dir=/opt/so/saltstack/local
-
-try_count=6
-interval=10
-
-while [[ $try_count -le 6 ]]; do
-    if docker top "so-playbook" &>/dev/null; then
-        automation_group=6
-
-        # Create user and retrieve api_key and user_id from response
-        mapfile -t automation_res < <(
-            curl -s --location --request POST 'http://127.0.0.1:3000/playbook/users.json' --user "admin:{{ admin_pass }}" --header 'Content-Type: application/json' --data '{
-                "user" : {
-                    "login" : "automation",
-                    "password": "{{ automation_pass }}",
-                    "firstname": "SecOps",
-                    "lastname": "Automation",
-                    "mail": "automation2@localhost.local"
-                }
-            }' | jq -r '.user.api_key, .user.id'
-        )
-
-        automation_api_key=${automation_res[0]}
-        automation_user_id=${automation_res[1]}
-
-        # Add user_id from newly created user to Automation group
-        curl -s --location --request POST "http://127.0.0.1:3000/playbook/groups/${automation_group}/users.json" \
-            --user "admin:{{ admin_pass }}" \
-            --header 'Content-Type: application/json' \
-            --data "{
-                \"user_id\" : ${automation_user_id}
-            }"
-
-        # Update the Automation API key in the secrets pillar
-        so-yaml.py remove $local_salt_dir/pillar/secrets.sls secrets.playbook_automation_api_key
-        printf '%s\n'\
-            "  playbook_automation_api_key: $automation_api_key" >> $local_salt_dir/pillar/secrets.sls
-        exit 0
-    fi
-    ((try_count++))
-    sleep "${interval}s"
-done
-
-# Timeout exceeded, exit with non-zero exit code
-exit 1

salt/playbook/files/playbook_db_init.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-# {%- set MYSQLPASS = salt['pillar.get']('secrets:mysql', None) -%}
-# {%- set admin_pass = salt['pillar.get']('secrets:playbook_admin', None) %}
-. /usr/sbin/so-common
-
-default_salt_dir=/opt/so/saltstack/default
-
-# Generate salt + hash for admin user
-admin_salt=$(get_random_value 32)
-admin_stage1_hash=$(echo -n '{{ admin_pass }}' | sha1sum | awk '{print $1}')
-admin_hash=$(echo -n "${admin_salt}${admin_stage1_hash}" | sha1sum | awk '{print $1}')
-sed -i "s/ADMIN_HASH/${admin_hash}/g" $default_salt_dir/salt/playbook/files/playbook_db_init.sql
-sed -i "s/ADMIN_SALT/${admin_salt}/g" $default_salt_dir/salt/playbook/files/playbook_db_init.sql
-
-# Copy file to destination + execute SQL
-docker cp $default_salt_dir/salt/playbook/files/playbook_db_init.sql so-mysql:/tmp/playbook_db_init.sql
-docker exec so-mysql /bin/bash -c "/usr/bin/mysql -b  -uroot -p{{MYSQLPASS}}  < /tmp/playbook_db_init.sql"