Merge pull request #13255 from Security-Onion-Solutions/2.4/dev

2.4.80

.github/.gitleaks.toml

@@ -536,11 +536,10 @@ secretGroup = 4
 
 [allowlist]
 description = "global allow lists"
-regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''']
+regexes = ['''219-09-9999''', '''078-05-1120''', '''(9[0-9]{2}|666)-\d{2}-\d{4}''', '''RPM-GPG-KEY.*''', '''.*:.*StrelkaHexDump.*''', '''.*:.*PLACEHOLDER.*''', '''ssl_.*password''']
 paths = [
     '''gitleaks.toml''',
     '''(.*?)(jpg|gif|doc|pdf|bin|svg|socket)$''',
     '''(go.mod|go.sum)$''',
-    
     '''salt/nginx/files/enterprise-attack.json'''
 ]

.github/DISCUSSION_TEMPLATE/2-4.yml

@@ -0,0 +1,190 @@
+body:
+  - type: markdown
+    attributes:
+      value: |
+        ⚠️ This category is solely for conversations related to Security Onion 2.4 ⚠️
+
+        If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
+  - type: dropdown
+    attributes:
+      label: Version
+      description: Which version of Security Onion 2.4.x are you asking about?
+      options:
+        -
+        - 2.4 Pre-release (Beta, Release Candidate)
+        - 2.4.10
+        - 2.4.20
+        - 2.4.30
+        - 2.4.40
+        - 2.4.50
+        - 2.4.60
+        - 2.4.70
+        - 2.4.80
+        - 2.4.90
+        - 2.4.100
+        - Other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Installation Method
+      description: How did you install Security Onion?
+      options:
+        -
+        - Security Onion ISO image
+        - Network installation on Red Hat derivative like Oracle, Rocky, Alma, etc.
+        - Network installation on Ubuntu
+        - Network installation on Debian
+        - Other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Description
+      description: >
+        Is this discussion about installation, configuration, upgrading, or other?
+      options:
+        -
+        - installation
+        - configuration
+        - upgrading
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Installation Type
+      description: >
+        When you installed, did you choose Import, Eval, Standalone, Distributed, or something else?
+      options:
+        -
+        - Import
+        - Eval
+        - Standalone
+        - Distributed
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Location
+      description: >
+        Is this deployment in the cloud, on-prem with Internet access, or airgap?
+      options:
+        -
+        - cloud
+        - on-prem with Internet access
+        - airgap
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Hardware Specs
+      description: >
+        Does your hardware meet or exceed the minimum requirements for your installation type as shown at https://docs.securityonion.net/en/2.4/hardware.html?
+      options:
+        -
+        - Meets minimum requirements
+        - Exceeds minimum requirements
+        - Does not meet minimum requirements
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: CPU
+      description: How many CPU cores do you have?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: RAM
+      description: How much RAM do you have?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Storage for /
+      description: How much storage do you have for the / partition?
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Storage for /nsm
+      description: How much storage do you have for the /nsm partition?
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Network Traffic Collection
+      description: >
+        Are you collecting network traffic from a tap or span port?
+      options:
+        -
+        - tap
+        - span port
+        - other (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Network Traffic Speeds
+      description: >
+        How much network traffic are you monitoring?
+      options:
+        -
+        - Less than 1Gbps
+        - 1Gbps to 10Gbps
+        - more than 10Gbps
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Status
+      description: >
+        Does SOC Grid show all services on all nodes as running OK?
+      options:
+        -
+        - Yes, all services on all nodes are running OK
+        - No, one or more services are failed (please provide detail below)
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Salt Status
+      description: >
+        Do you get any failures when you run "sudo salt-call state.highstate"?
+      options:
+        -
+        - Yes, there are salt failures (please provide detail below)
+        - No, there are no failures
+    validations:
+      required: true
+  - type: dropdown
+    attributes:
+      label: Logs
+      description: >
+        Are there any additional clues in /opt/so/log/?
+      options:
+        -
+        - Yes, there are additional clues in /opt/so/log/ (please provide detail below)
+        - No, there are no additional clues
+    validations:
+      required: true
+  - type: textarea
+    attributes:
+      label: Detail
+      description: Please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and then provide detailed information to help us help you.
+      placeholder: |-
+        STOP! Before typing, please read our discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 in their entirety!
+
+        If your organization needs more immediate, enterprise grade professional support, with one-on-one virtual meetings and screensharing, contact us via our website: https://securityonion.com/support
+    validations:
+      required: true
+  - type: checkboxes
+    attributes:
+      label: Guidelines
+      options:
+        - label: I have read the discussion guidelines at https://github.com/Security-Onion-Solutions/securityonion/discussions/1720 and assert that I have followed the guidelines.
+          required: true

.github/workflows/close-threads.yml

@@ -0,0 +1,33 @@
+name: 'Close Threads'
+
+on:
+  schedule:
+    - cron: '50 1 * * *'
+  workflow_dispatch:
+
+permissions:
+  issues: write
+  pull-requests: write
+  discussions: write
+
+concurrency:
+  group: lock-threads
+
+jobs:
+  close-threads:
+    if: github.repository_owner == 'security-onion-solutions'
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v5
+        with:
+          days-before-issue-stale: -1
+          days-before-issue-close: 60
+          stale-issue-message: "This issue is stale because it has been inactive for an extended period. Stale issues convey that the issue, while important to someone, is not critical enough for the author, or other community members to work on, sponsor, or otherwise shepherd the issue through to a resolution."
+          close-issue-message: "This issue was closed because it has been stale for an extended period. It will be automatically locked in 30 days, after which no further commenting will be available."
+          days-before-pr-stale: 45
+          days-before-pr-close: 60
+          stale-pr-message: "This PR is stale because it has been inactive for an extended period. The longer a PR remains stale the more out of date with the main branch it becomes."
+          close-pr-message: "This PR was closed because it has been stale for an extended period. It will be automatically locked in 30 days. If there is still a commitment to finishing this PR re-open it before it is locked."

.github/workflows/contrib.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - name: "Contributor Check"
         if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: cla-assistant/github-action@v2.1.3-beta
+        uses: cla-assistant/github-action@v2.3.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}

.github/workflows/lock-threads.yml

@@ -0,0 +1,26 @@
+name: 'Lock Threads'
+
+on:
+  schedule:
+    - cron: '50 2 * * *'
+  workflow_dispatch:
+
+permissions:
+  issues: write
+  pull-requests: write
+  discussions: write
+
+concurrency:
+  group: lock-threads
+
+jobs:
+  lock-threads:
+    if: github.repository_owner == 'security-onion-solutions'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: jertel/lock-threads@main
+        with:
+          include-discussion-currently-open: true
+          discussion-inactive-days: 90
+          issue-inactive-days: 30
+          pr-inactive-days: 30

.github/workflows/pythontest.yml

@@ -4,9 +4,11 @@ on:
   push:
     paths: 
       - "salt/sensoroni/files/analyzers/**"
+      - "salt/manager/tools/sbin"
   pull_request:
     paths:
       - "salt/sensoroni/files/analyzers/**"
+      - "salt/manager/tools/sbin"
 
 jobs:
   build:
@@ -16,7 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.10"]
-        python-code-path: ["salt/sensoroni/files/analyzers"]
+        python-code-path: ["salt/sensoroni/files/analyzers", "salt/manager/tools/sbin"]
 
     steps:
     - uses: actions/checkout@v3
@@ -34,4 +36,4 @@ jobs:
         flake8 ${{ matrix.python-code-path }} --show-source --max-complexity=12  --doctests --max-line-length=200 --statistics
     - name: Test with pytest
       run: |
-        pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=${{ matrix.python-code-path }}/pytest.ini
+        pytest ${{ matrix.python-code-path }} --cov=${{ matrix.python-code-path }} --doctest-modules --cov-report=term --cov-fail-under=100 --cov-config=pytest.ini

DOWNLOAD_AND_VERIFY_ISO.md

@@ -0,0 +1,53 @@
+### 2.4.80-20240624 ISO image released on 2024/06/25
+
+
+### Download and Verify
+
+2.4.80-20240624 ISO image:  
+https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso
+ 
+MD5: 139F9762E926F9CB3C4A9528A3752C31  
+SHA1: BC6CA2C5F4ABC1A04E83A5CF8FFA6A53B1583CC9  
+SHA256: 70E90845C84FFA30AD6CF21504634F57C273E7996CA72F7250428DDBAAC5B1BD  
+
+Signature for ISO image:  
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig
+
+Signing key:  
+https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
+
+For example, here are the steps you can use on most Linux distributions to download and verify our Security Onion ISO image.
+
+Download and import the signing key:  
+```
+wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS -O - | gpg --import -  
+```
+
+Download the signature file for the ISO:  
+```
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig
+```
+
+Download the ISO image:  
+```
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso
+```
+
+Verify the downloaded ISO image using the signature file:  
+```
+gpg --verify securityonion-2.4.80-20240624.iso.sig securityonion-2.4.80-20240624.iso
+```
+
+The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
+```
+gpg: Signature made Mon 24 Jun 2024 02:42:03 PM EDT using RSA key ID FE507013
+gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
+gpg: WARNING: This key is not certified with a trusted signature!
+gpg:          There is no indication that the signature belongs to the owner.
+Primary key fingerprint: C804 A93D 36BE 0C73 3EA1  9644 7C10 60B7 FE50 7013
+```
+
+If it fails to verify, try downloading again. If it still fails to verify, try downloading from another computer or another network.
+
+Once you've verified the ISO image, you're ready to proceed to our Installation guide:  
+https://docs.securityonion.net/en/2.4/installation.html

HOTFIX

@@ -1 +0,0 @@
-

README.md

@@ -1,20 +1,29 @@
-## Security Onion 2.4 Beta 2
+## Security Onion 2.4
 
-Security Onion 2.4 Beta 2 is here!
+Security Onion 2.4 is here!
 
 ## Screenshots
 
 Alerts
-![Alerts](./assets/images/screenshots/alerts.png)
+![Alerts](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/50_alerts.png)
 
 Dashboards
-![Dashboards](./assets/images/screenshots/dashboards.png)
+![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_dashboards.png)
 
 Hunt
-![Hunt](./assets/images/screenshots/hunt.png)
+![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/56_hunt.png)
 
-Cases
-![Cases](./assets/images/screenshots/cases-comments.png)
+Detections
+![Detections](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_detections.png)
+
+PCAP
+![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/62_pcap.png)
+
+Grid
+![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/75_grid.png)
+
+Config
+![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/87_config.png)
 
 ### Release Notes
 

VERIFY_ISO.md

@@ -1 +0,0 @@
-### An ISO will be available starting in RC1. 

VERSION

@@ -1 +1 @@
-2.4.1
+2.4.80

files/firewall/assigned_hostgroups.local.map.yaml

@@ -12,7 +12,6 @@ role:
   eval:
   fleet:
   heavynode:
-  helixsensor:
   idh:
   import:
   manager:
@@ -20,4 +19,4 @@ role:
   receiver:
   standalone:
   searchnode:
-  sensor:
+  sensor:

files/salt/master/master

@@ -41,7 +41,8 @@ file_roots:
   base:
     - /opt/so/saltstack/local/salt
     - /opt/so/saltstack/default/salt
-
+    - /nsm/elastic-fleet/artifacts
+    - /opt/so/rules/nids
 
 # The master_roots setting configures a master-only copy of the file_roots dictionary,
 # used by the state compiler.

pillar/kafka/nodes.sls

@@ -0,0 +1,2 @@
+kafka:
+  nodes:

pillar/logrotate/init.sls

@@ -1,13 +0,0 @@
-logrotate:
-  conf: | 
-    daily
-    rotate 14
-    missingok
-    copytruncate
-    compress
-    create
-    extension .log
-    dateext
-    dateyesterday
-  group_conf: |
-    su root socore

pillar/logstash/fleet.sls

@@ -1,6 +0,0 @@
-logstash:
-  pipelines:
-    fleet:
-      config:
-        - so/0012_input_elastic_agent.conf     
-        - so/9806_output_lumberjack_fleet.conf.jinja

pillar/logstash/helix.sls

@@ -1,42 +0,0 @@
-logstash:
-  pipelines:
-    helix:
-      config:
-        - so/0010_input_hhbeats.conf
-        - so/1033_preprocess_snort.conf
-        - so/1100_preprocess_bro_conn.conf
-        - so/1101_preprocess_bro_dhcp.conf
-        - so/1102_preprocess_bro_dns.conf
-        - so/1103_preprocess_bro_dpd.conf
-        - so/1104_preprocess_bro_files.conf
-        - so/1105_preprocess_bro_ftp.conf
-        - so/1106_preprocess_bro_http.conf
-        - so/1107_preprocess_bro_irc.conf
-        - so/1108_preprocess_bro_kerberos.conf
-        - so/1109_preprocess_bro_notice.conf
-        - so/1110_preprocess_bro_rdp.conf
-        - so/1111_preprocess_bro_signatures.conf
-        - so/1112_preprocess_bro_smtp.conf
-        - so/1113_preprocess_bro_snmp.conf
-        - so/1114_preprocess_bro_software.conf
-        - so/1115_preprocess_bro_ssh.conf
-        - so/1116_preprocess_bro_ssl.conf
-        - so/1117_preprocess_bro_syslog.conf
-        - so/1118_preprocess_bro_tunnel.conf
-        - so/1119_preprocess_bro_weird.conf
-        - so/1121_preprocess_bro_mysql.conf
-        - so/1122_preprocess_bro_socks.conf
-        - so/1123_preprocess_bro_x509.conf
-        - so/1124_preprocess_bro_intel.conf
-        - so/1125_preprocess_bro_modbus.conf
-        - so/1126_preprocess_bro_sip.conf
-        - so/1127_preprocess_bro_radius.conf
-        - so/1128_preprocess_bro_pe.conf
-        - so/1129_preprocess_bro_rfb.conf
-        - so/1130_preprocess_bro_dnp3.conf
-        - so/1131_preprocess_bro_smb_files.conf
-        - so/1132_preprocess_bro_smb_mapping.conf
-        - so/1133_preprocess_bro_ntlm.conf
-        - so/1134_preprocess_bro_dce_rpc.conf
-        - so/8001_postprocess_common_ip_augmentation.conf
-        - so/9997_output_helix.conf.jinja

pillar/logstash/manager.sls

@@ -1,8 +0,0 @@
-logstash:
-  pipelines:
-    manager:
-      config:
-        - so/0011_input_endgame.conf
-        - so/0012_input_elastic_agent.conf
-        - so/0013_input_lumberjack_fleet.conf
-        - so/9999_output_redis.conf.jinja

pillar/logstash/nodes.sls

@@ -2,24 +2,28 @@
 {% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %}
 {% for minionid, ip in salt.saltutil.runner(
     'mine.get',
-    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ',
+    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ',
     fun='network.ip_addrs',
     tgt_type='compound') | dictsort()
 %}
 
-{%   set hostname = cached_grains[minionid]['host'] %}
-{%   set node_type = minionid.split('_')[1] %}
-{%   if node_type not in node_types.keys() %}
-{%     do node_types.update({node_type: {hostname: ip[0]}}) %}
-{%   else %}
-{%     if hostname not in node_types[node_type] %}
-{%       do node_types[node_type].update({hostname: ip[0]}) %}
+# only add a node to the pillar if it returned an ip from the mine
+{%   if ip | length > 0%}
+{%     set hostname = cached_grains[minionid]['host'] %}
+{%     set node_type = minionid.split('_')[1] %}
+{%     if node_type not in node_types.keys() %}
+{%       do node_types.update({node_type: {hostname: ip[0]}}) %}
 {%     else %}
-{%       do node_types[node_type][hostname].update(ip[0]) %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: ip[0]}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update(ip[0]) %}
+{%       endif %}
 {%     endif %}
 {%   endif %}
 {% endfor %}
 
+
 logstash:
   nodes:
 {% for node_type, values in node_types.items() %}

pillar/logstash/receiver.sls

@@ -1,8 +0,0 @@
-logstash:
-  pipelines:
-    receiver:
-      config:
-        - so/0011_input_endgame.conf
-        - so/0012_input_elastic_agent.conf
-        - so/9999_output_redis.conf.jinja
-        

pillar/logstash/search.sls

@@ -1,7 +0,0 @@
-logstash:
-  pipelines:
-    search:
-      config:
-        - so/0900_input_redis.conf.jinja
-        - so/9805_output_elastic_agent.conf.jinja
-        - so/9900_output_endgame.conf.jinja

pillar/node_data/ips.sls

@@ -4,18 +4,22 @@
 {%   set hostname = minionid.split('_')[0] %}
 {%   set node_type = minionid.split('_')[1] %}
 {%   set is_alive = False %}
-{%   if minionid in manage_alived.keys() %}
-{%     if ip[0] == manage_alived[minionid] %}
-{%       set is_alive = True %}
+
+# only add a node to the pillar if it returned an ip from the mine
+{%   if ip | length > 0%}
+{%     if minionid in manage_alived.keys() %}
+{%       if ip[0] == manage_alived[minionid] %}
+{%         set is_alive = True %}
+{%       endif %}
 {%     endif %}
-{%   endif %}
-{%   if node_type not in node_types.keys() %}
-{%     do node_types.update({node_type: {hostname: {'ip':ip[0], 'alive':is_alive }}}) %}
-{%   else %}
-{%     if hostname not in node_types[node_type] %}
-{%       do node_types[node_type].update({hostname: {'ip':ip[0], 'alive':is_alive}}) %}
+{%     if node_type not in node_types.keys() %}
+{%       do node_types.update({node_type: {hostname: {'ip':ip[0], 'alive':is_alive }}}) %}
 {%     else %}
-{%       do node_types[node_type][hostname].update({'ip':ip[0], 'alive':is_alive}) %}
+{%       if hostname not in node_types[node_type] %}
+{%         do node_types[node_type].update({hostname: {'ip':ip[0], 'alive':is_alive}}) %}
+{%       else %}
+{%         do node_types[node_type][hostname].update({'ip':ip[0], 'alive':is_alive}) %}
+{%       endif %}
 {%     endif %}
 {%   endif %}
 {% endfor %}

pillar/soc/license.sls

@@ -0,0 +1,14 @@
+# Copyright Jason Ertel (github.com/jertel).
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with 
+# the Elastic License 2.0.
+
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+# This file is generated by Security Onion and contains a list of license-enabled features. 
+features: []

pillar/thresholding/pillar.example

@@ -1,44 +0,0 @@
-thresholding:
-  sids:
-    8675309:
-      - threshold:
-          gen_id: 1
-          type: threshold
-          track: by_src
-          count: 10
-          seconds: 10
-      - threshold:
-          gen_id: 1
-          type: limit
-          track: by_dst
-          count: 100
-          seconds: 30
-      - rate_filter:
-          gen_id: 1
-          track: by_rule
-          count: 50
-          seconds: 30
-          new_action: alert
-          timeout: 30
-      - suppress:
-          gen_id: 1
-          track: by_either
-          ip: 10.10.3.7
-    11223344:
-      - threshold:
-          gen_id: 1
-          type: limit
-          track: by_dst
-          count: 10
-          seconds: 10
-      - rate_filter:
-          gen_id: 1
-          track: by_src
-          count: 50
-          seconds: 20
-          new_action: pass
-          timeout: 60
-      - suppress:
-          gen_id: 1
-          track: by_src
-          ip: 10.10.3.0/24

pillar/thresholding/pillar.usage

@@ -1,20 +0,0 @@
-thresholding:
-  sids:
-    <signature id>:
-      - threshold:
-          gen_id: <generator id>
-          type: <threshold | limit | both>
-          track: <by_src | by_dst>
-          count: <count>
-          seconds: <seconds>
-      - rate_filter:
-          gen_id: <generator id>
-          track: <by_src | by_dst | by_rule | by_both>
-          count: <count>
-          seconds: <seconds>
-          new_action: <alert | pass>
-          timeout: <seconds>
-      - suppress:
-          gen_id: <generator id>
-          track: <by_src | by_dst | by_either>
-          ip: <ip | subnet>

pillar/top.sls

@@ -1,45 +1,29 @@
 base:
   '*':
-    - patch.needs_restarting
-    - ntp.soc_ntp
-    - ntp.adv_ntp
-    - logrotate
+    - global.soc_global
+    - global.adv_global
     - docker.soc_docker
     - docker.adv_docker
+    - influxdb.token
+    - logrotate.soc_logrotate
+    - logrotate.adv_logrotate
+    - ntp.soc_ntp
+    - ntp.adv_ntp
+    - patch.needs_restarting
+    - patch.soc_patch
+    - patch.adv_patch
     - sensoroni.soc_sensoroni
     - sensoroni.adv_sensoroni
     - telegraf.soc_telegraf
     - telegraf.adv_telegraf
-    - influxdb.token
+
+  '* and not *_desktop':
+    - firewall.soc_firewall
+    - firewall.adv_firewall
+    - nginx.soc_nginx
+    - nginx.adv_nginx
     - node_data.ips
 
-  '* and not *_eval and not *_import':
-    - logstash.nodes
-
-  '*_eval or *_heavynode or *_sensor or *_standalone or *_import':
-    - match: compound
-    - zeek
-    - bpf.soc_bpf
-    - bpf.adv_bpf
-
-  '*_managersearch or *_heavynode':
-    - match: compound
-    - logstash
-    - logstash.manager
-    - logstash.search
-    - logstash.soc_logstash
-    - logstash.adv_logstash
-    - elasticsearch.index_templates
-    - elasticsearch.soc_elasticsearch
-    - elasticsearch.adv_elasticsearch
-
-  '*_manager':
-    - logstash
-    - logstash.manager
-    - logstash.soc_logstash
-    - logstash.adv_logstash
-    - elasticsearch.index_templates
-
   '*_manager or *_managersearch':
     - match: compound
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
@@ -49,14 +33,18 @@ base:
     - kibana.secrets
     {% endif %}
     - secrets
-    - global.soc_global
-    - global.adv_global
     - manager.soc_manager
     - manager.adv_manager
     - idstools.soc_idstools
     - idstools.adv_idstools
+    - logstash.nodes
+    - logstash.soc_logstash
+    - logstash.adv_logstash
     - soc.soc_soc
     - soc.adv_soc
+    - soc.license
+    - kibana.soc_kibana
+    - kibana.adv_kibana
     - kratos.soc_kratos
     - kratos.adv_kratos
     - redis.soc_redis
@@ -65,19 +53,35 @@ base:
     - influxdb.adv_influxdb
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
+    - elastalert.soc_elastalert
+    - elastalert.adv_elastalert
     - backup.soc_backup
     - backup.adv_backup
-    - firewall.soc_firewall
-    - firewall.adv_firewall
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - kafka.nodes
+    - kafka.soc_kafka
+    - kafka.adv_kafka
+    - stig.soc_stig
 
   '*_sensor':
     - healthcheck.sensor
-    - global.soc_global
-    - global.adv_global
+    - strelka.soc_strelka
+    - strelka.adv_strelka
+    - zeek.soc_zeek
+    - zeek.adv_zeek
+    - bpf.soc_bpf
+    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
+    - suricata.soc_suricata
+    - suricata.adv_suricata
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - stig.soc_stig
+    - soc.license
 
   '*_eval':
     - secrets
@@ -89,16 +93,24 @@ base:
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
-    - global.soc_global
-    - global.adv_global
     - kratos.soc_kratos
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
+    - elastalert.soc_elastalert
+    - elastalert.adv_elastalert
     - manager.soc_manager
     - manager.adv_manager
     - idstools.soc_idstools
     - idstools.adv_idstools
     - soc.soc_soc
+    - soc.adv_soc
+    - soc.license
+    - kibana.soc_kibana
+    - kibana.adv_kibana
+    - strelka.soc_strelka
+    - strelka.adv_strelka
     - kratos.soc_kratos
     - kratos.adv_kratos
     - redis.soc_redis
@@ -107,15 +119,19 @@ base:
     - influxdb.adv_influxdb
     - backup.soc_backup
     - backup.adv_backup
-    - firewall.soc_firewall
-    - firewall.adv_firewall
+    - zeek.soc_zeek
+    - zeek.adv_zeek
+    - bpf.soc_bpf
+    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
+    - suricata.soc_suricata
+    - suricata.adv_suricata
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
   '*_standalone':
-    - logstash
-    - logstash.manager
-    - logstash.search
+    - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
     - elasticsearch.index_templates
@@ -127,8 +143,6 @@ base:
     {% endif %}
     - secrets
     - healthcheck.standalone
-    - global.soc_global
-    - global.adv_global
     - idstools.soc_idstools
     - idstools.adv_idstools
     - kratos.soc_kratos
@@ -139,52 +153,83 @@ base:
     - influxdb.adv_influxdb
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
+    - elastalert.soc_elastalert
+    - elastalert.adv_elastalert
     - manager.soc_manager
     - manager.adv_manager
     - soc.soc_soc
+    - soc.adv_soc
+    - soc.license
+    - kibana.soc_kibana
+    - kibana.adv_kibana
+    - strelka.soc_strelka
+    - strelka.adv_strelka
     - backup.soc_backup
     - backup.adv_backup
-    - firewall.soc_firewall
-    - firewall.adv_firewall
+    - zeek.soc_zeek
+    - zeek.adv_zeek
+    - bpf.soc_bpf
+    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
+    - suricata.soc_suricata
+    - suricata.adv_suricata
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - stig.soc_stig
+    - kafka.nodes
+    - kafka.soc_kafka
+    - kafka.adv_kafka
 
   '*_heavynode':
     - elasticsearch.auth
-    - global.soc_global
-    - global.adv_global
+    - logstash.nodes
+    - logstash.soc_logstash
+    - logstash.adv_logstash
+    - elasticsearch.soc_elasticsearch
+    - elasticsearch.adv_elasticsearch
     - redis.soc_redis
+    - redis.adv_redis
+    - zeek.soc_zeek
+    - zeek.adv_zeek
+    - bpf.soc_bpf
+    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
+    - suricata.soc_suricata
+    - suricata.adv_suricata
+    - strelka.soc_strelka
+    - strelka.adv_strelka
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
   '*_idh':
-    - global.soc_global
-    - global.adv_global
     - idh.soc_idh
     - idh.adv_idh
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
   '*_searchnode':
-    - logstash
-    - logstash.search
+    - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
-    - elasticsearch.index_templates
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
     - redis.soc_redis
-    - global.soc_global
-    - global.adv_global
+    - redis.adv_redis
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - stig.soc_stig
+    - soc.license
+    - kafka.nodes
 
   '*_receiver':
-    - logstash
-    - logstash.receiver
+    - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
@@ -192,10 +237,12 @@ base:
     {% endif %}
     - redis.soc_redis
     - redis.adv_redis
-    - global.soc_global
-    - global.adv_global
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
+    - kafka.nodes
+    - kafka.soc_kafka
+    - kafka.adv_kafka
+    - soc.license
 
   '*_import':
     - secrets
@@ -209,11 +256,17 @@ base:
     - kratos.soc_kratos
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
+    - elastalert.soc_elastalert
+    - elastalert.adv_elastalert
     - manager.soc_manager
     - manager.adv_manager
     - soc.soc_soc
-    - global.soc_global
-    - global.adv_global
+    - soc.adv_soc
+    - soc.license
+    - kibana.soc_kibana
+    - kibana.adv_kibana
     - backup.soc_backup
     - backup.adv_backup
     - kratos.soc_kratos
@@ -222,23 +275,30 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
-    - firewall.soc_firewall
-    - firewall.adv_firewall
+    - zeek.soc_zeek
+    - zeek.adv_zeek
+    - bpf.soc_bpf
+    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
+    - suricata.soc_suricata
+    - suricata.adv_suricata
+    - strelka.soc_strelka
+    - strelka.adv_strelka
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
   '*_fleet':
-    - global.soc_global
-    - global.adv_global
     - backup.soc_backup
     - backup.adv_backup
-    - logstash
-    - logstash.fleet
+    - logstash.nodes
     - logstash.soc_logstash
     - logstash.adv_logstash
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
-  '*_workstation':
+  '*_desktop':
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}

pyci.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+if [[ $# -ne 1 ]]; then
+    echo "Usage: $0 <python_script_dir>"
+    echo "Runs tests on all *_test.py files in the given directory."
+    exit 1
+fi
+
+HOME_DIR=$(dirname "$0")
+TARGET_DIR=${1:-.}
+
+PATH=$PATH:/usr/local/bin
+
+if [ ! -d .venv ]; then
+    python -m venv .venv
+fi
+
+source .venv/bin/activate
+
+if ! pip install flake8 pytest pytest-cov pyyaml; then
+    echo "Unable to install dependencies."
+    exit 1
+fi
+
+flake8 "$TARGET_DIR" "--config=${HOME_DIR}/pytest.ini"
+python3 -m pytest "--cov-config=${HOME_DIR}/pytest.ini" "--cov=$TARGET_DIR" --doctest-modules --cov-report=term --cov-fail-under=100  "$TARGET_DIR" 

salt/_modules/needs_restarting.py

@@ -3,14 +3,14 @@ import subprocess
 
 def check():
 
-  os = __grains__['os']
+  osfam = __grains__['os_family']
   retval = 'False'
 
-  if os == 'Ubuntu':
+  if osfam == 'Debian':
     if path.exists('/var/run/reboot-required'):
       retval = 'True'
 
-  elif os == 'Rocky':
+  elif osfam == 'RedHat':
     cmd = 'needs-restarting -r > /dev/null 2>&1'
     
     try:

salt/allowed_states.map.jinja

@@ -3,16 +3,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
-{% set ZEEKVER = salt['pillar.get']('global:mdengine', '') %}
-{% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
-{% set ELASTALERT = salt['pillar.get']('elastalert:enabled', True) %}
-{% set ELASTICSEARCH = salt['pillar.get']('elasticsearch:enabled', True) %}
-{% set KIBANA = salt['pillar.get']('kibana:enabled', True) %}
-{% set LOGSTASH = salt['pillar.get']('logstash:enabled', True) %}
-{% set CURATOR = salt['pillar.get']('curator:enabled', True) %}
-{% set REDIS = salt['pillar.get']('redis:enabled', True) %}
-{% set STRELKA = salt['pillar.get']('strelka:enabled', '0') %}
 {% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %}
 {% import_yaml 'salt/minion.defaults.yaml' as saltversion %}
 {% set saltversion = saltversion.salt.minion.version %}
@@ -35,6 +25,7 @@
           'soc',
           'kratos',
           'elasticfleet',
+          'elastic-fleet-package-registry',
           'firewall',
           'idstools',
           'suricata.manager',
@@ -43,7 +34,6 @@
           'suricata',
           'utility',
           'schedule',
-          'soctopus',
           'tcpreplay',
           'docker_clean'
           ],
@@ -55,23 +45,7 @@
           'pcap',
           'suricata',
           'healthcheck',
-          'schedule',
-          'tcpreplay',
-          'docker_clean'
-          ],
-      'so-helixsensor': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'telegraf',
-          'firewall',
-          'idstools',
-          'suricata.manager',
-          'zeek',
-          'redis',
-          'elasticsearch',
-          'logstash',
+          'elasticagent',
           'schedule',
           'tcpreplay',
           'docker_clean'
@@ -91,6 +65,7 @@
           'registry',
           'manager',
           'nginx',
+          'strelka.manager',
           'soc',
           'kratos',
           'influxdb',
@@ -105,7 +80,8 @@
           'schedule',
           'tcpreplay',
           'docker_clean',
-          'elasticfleet'
+          'elasticfleet',
+          'elastic-fleet-package-registry'
           ],
       'so-manager': [
           'salt.master',
@@ -116,16 +92,19 @@
           'nginx',
           'telegraf',
           'influxdb',
+          'strelka.manager',
           'soc',
           'kratos',
           'elasticfleet',
+          'elastic-fleet-package-registry',
           'firewall',
           'idstools',
           'suricata.manager',
           'utility',
           'schedule',
-          'soctopus',
-          'docker_clean'
+          'docker_clean',
+          'stig',
+          'kafka'
           ],
       'so-managersearch': [
           'salt.master',
@@ -135,8 +114,10 @@
           'nginx',
           'telegraf',
           'influxdb',
+          'strelka.manager',
           'soc',
           'kratos',
+          'elastic-fleet-package-registry',
           'elasticfleet',
           'firewall',
           'manager',
@@ -144,8 +125,9 @@
           'suricata.manager',
           'utility',
           'schedule',
-          'soctopus',
-          'docker_clean'
+          'docker_clean',
+          'stig',
+          'kafka'
           ],
       'so-searchnode': [
           'ssl',
@@ -153,7 +135,8 @@
           'telegraf',
           'firewall',
           'schedule',
-          'docker_clean'
+          'docker_clean',
+          'stig'
           ],
       'so-standalone': [
           'salt.master',
@@ -166,6 +149,7 @@
           'influxdb',
           'soc',
           'kratos',
+          'elastic-fleet-package-registry',
           'elasticfleet',
           'firewall',
           'idstools',
@@ -175,9 +159,10 @@
           'healthcheck',
           'utility',
           'schedule',
-          'soctopus',
           'tcpreplay',
-          'docker_clean'
+          'docker_clean',
+          'stig',
+          'kafka'
           ],
       'so-sensor': [
           'ssl',
@@ -189,13 +174,15 @@
           'healthcheck',
           'schedule',
           'tcpreplay',
-          'docker_clean'
+          'docker_clean',
+          'stig'
           ],
       'so-fleet': [
           'ssl',
           'telegraf',
           'firewall',
           'logstash',
+          'nginx',
           'healthcheck',
           'schedule',
           'elasticfleet',
@@ -206,58 +193,48 @@
           'telegraf',
           'firewall',
           'schedule',
-          'docker_clean'
+          'docker_clean',
+          'kafka',
+          'elasticsearch.ca',
+          'stig'
           ],
-      'so-workstation': [
+      'so-desktop': [
+          'ssl',
+          'docker_clean',
+          'telegraf'
           ],
   }, grain='role') %}
 
-  {% if (PLAYBOOK != 0) and grains.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone'] %}
-    {% do allowed_states.append('mysql') %}
-  {% endif %}
-
-  {%- if ZEEKVER != 'SURICATA' and grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
+  {%- if grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
     {% do allowed_states.append('zeek') %}
   {%- endif %}
 
-  {% if STRELKA and grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
+  {% if grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
     {% do allowed_states.append('strelka') %}
   {% endif %}
 
-  {% if ELASTICSEARCH and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import'] %}
+  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import'] %}
     {% do allowed_states.append('elasticsearch') %}
   {% endif %}
 
-  {% if ELASTICSEARCH and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
+  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
     {% do allowed_states.append('elasticsearch.auth') %}
   {% endif %}
 
-  {% if KIBANA and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
+  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
     {% do allowed_states.append('kibana') %}
     {% do allowed_states.append('kibana.secrets') %}
   {% endif %}
 
-  {% if grains.role in ['so-eval', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-manager'] %}
-    {% do allowed_states.append('curator') %}
-  {% endif %}
-
-  {% if ELASTALERT and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
+  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
     {% do allowed_states.append('elastalert') %}
   {% endif %}
 
-  {% if (PLAYBOOK !=0) and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
-    {% do allowed_states.append('playbook') %}
-  {% endif %}
-
-  {% if (PLAYBOOK !=0) and grains.role in ['so-eval'] %}
-    {% do allowed_states.append('redis') %}
-  {% endif %}
-
-  {% if LOGSTASH and grains.role in ['so-helixsensor', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
+  {% if grains.role in ['so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
     {% do allowed_states.append('logstash') %}
   {% endif %}
 
-  {% if REDIS and grains.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
+  {% if grains.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-receiver', 'so-eval'] %}
     {% do allowed_states.append('redis') %}
   {% endif %}
   

salt/bpf/macros.jinja

@@ -0,0 +1,10 @@
+{% macro remove_comments(bpfmerged, app) %}
+
+{# remove comments from the bpf #}
+{% for bpf in bpfmerged[app] %}
+{%   if bpf.strip().startswith('#') %}
+{%     do bpfmerged[app].pop(loop.index0) %}
+{%   endif %}
+{% endfor %}
+
+{% endmacro %}

salt/bpf/pcap.map.jinja

@@ -1,4 +1,10 @@
-{% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
-{% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
-
-{% set PCAPBPF = BPFMERGED.pcap %}
+{% from 'vars/globals.map.jinja' import GLOBALS %}
+{% if GLOBALS.pcap_engine == "TRANSITION" %}
+{%   set PCAPBPF = ["ip and host 255.255.255.1 and port 1"] %}
+{% else %}
+{%   import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
+{%   set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
+{%   import 'bpf/macros.jinja' as MACROS %}
+{{   MACROS.remove_comments(BPFMERGED, 'pcap') }}
+{%   set PCAPBPF = BPFMERGED.pcap %}
+{% endif %}

salt/bpf/soc_bpf.yaml

@@ -1,6 +1,6 @@
 bpf:
   pcap:
-    description: List of BPF filters to apply to PCAP.
+    description: List of BPF filters to apply to Stenographer.
     multiline: True
     forcedType: "[]string"
     helpLink: bpf.html

salt/bpf/suricata.map.jinja

@@ -1,4 +1,7 @@
 {% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
+{% import 'bpf/macros.jinja' as MACROS %}
+
+{{ MACROS.remove_comments(BPFMERGED, 'suricata') }}
 
 {% set SURICATABPF = BPFMERGED.suricata %}

salt/bpf/zeek.map.jinja

@@ -1,4 +1,7 @@
 {% import_yaml 'bpf/defaults.yaml' as BPFDEFAULTS %}
 {% set BPFMERGED = salt['pillar.get']('bpf', BPFDEFAULTS.bpf, merge=True) %}
+{% import 'bpf/macros.jinja' as MACROS %}
+
+{{ MACROS.remove_comments(BPFMERGED, 'zeek') }}
 
 {% set ZEEKBPF = BPFMERGED.zeek %}

salt/ca/files/signing_policies.conf

@@ -1,6 +1,3 @@
-mine_functions:
-  x509.get_pem_entries: [/etc/pki/ca.crt]
-
 x509_signing_policies:
   filebeat:
     - minions: '*'
@@ -37,7 +34,7 @@ x509_signing_policies:
     - ST: Utah
     - L: Salt Lake City
     - basicConstraints: "critical CA:false"
-    - keyUsage: "critical keyEncipherment"
+    - keyUsage: "critical keyEncipherment digitalSignature"
     - subjectKeyIdentifier: hash
     - authorityKeyIdentifier: keyid,issuer:always
     - extendedKeyUsage: serverAuth
@@ -70,3 +67,17 @@ x509_signing_policies:
     - authorityKeyIdentifier: keyid,issuer:always
     - days_valid: 820
     - copypath: /etc/pki/issued_certs/
+  kafka:
+    - minions: '*'
+    - signing_private_key: /etc/pki/ca.key
+    - signing_cert: /etc/pki/ca.crt
+    - C: US
+    - ST: Utah
+    - L: Salt Lake City
+    - basicConstraints: "critical CA:false"
+    - keyUsage: "digitalSignature, keyEncipherment"
+    - subjectKeyIdentifier: hash
+    - authorityKeyIdentifier: keyid,issuer:always
+    - extendedKeyUsage: "serverAuth, clientAuth"
+    - days_valid: 820
+    - copypath: /etc/pki/issued_certs/

salt/ca/init.sls

@@ -20,7 +20,6 @@ pki_private_key:
     - name: /etc/pki/ca.key
     - keysize: 4096
     - passphrase:
-    - cipher: aes_256_cbc
     - backup: True
     {% if salt['file.file_exists']('/etc/pki/ca.key') -%}
     - prereq:
@@ -51,6 +50,12 @@ pki_public_ca_crt:
         attempts: 5
         interval: 30
 
+mine_update_ca_crt:
+  module.run:
+    - mine.update: []
+    - onchanges:
+      - x509: pki_public_ca_crt
+
 cakeyperms:
   file.managed:
     - replace: False

salt/common/cron/common-rotate

@@ -1,2 +0,0 @@
-#!/bin/bash
-/usr/sbin/logrotate -f /opt/so/conf/log-rotate.conf > /dev/null 2>&1

salt/common/cron/sensor-rotate

@@ -1,2 +0,0 @@
-#!/bin/bash
-/usr/sbin/logrotate -f /opt/so/conf/sensor-rotate.conf > /dev/null 2>&1

salt/common/files/analyst/README

@@ -1,79 +0,0 @@
-The following GUI tools are available on the analyst workstation:
-
-chromium
-  url: https://www.chromium.org/Home
-  To run chromium, click Applications > Internet > Chromium Web Browser
-  
-Wireshark
-  url: https://www.wireshark.org/
-  To run Wireshark, click Applications > Internet > Wireshark Network Analyzer
-
-NetworkMiner
-  url: https://www.netresec.com
-  To run NetworkMiner, click Applications > Internet > NetworkMiner
-
-The following CLI tools are available on the analyst workstation:
-
-bit-twist
-  url: http://bittwist.sourceforge.net
-  To run bit-twist, open a terminal and type: bittwist -h
-
-chaosreader
-  url: http://chaosreader.sourceforge.net
-  To run chaosreader, open a terminal and type: chaosreader -h
-
-dnsiff
-  url: https://www.monkey.org/~dugsong/dsniff/
-  To run dsniff, open a terminal and type: dsniff -h
-
-foremost
-  url: http://foremost.sourceforge.net
-  To run foremost, open a terminal and type: foremost -h
-  
-hping3
-  url: http://www.hping.org/hping3.html
-  To run hping3, open a terminal and type: hping3 -h
-
-netsed
-  url: http://silicone.homelinux.org/projects/netsed/
-  To run netsed, open a terminal and type: netsed -h
-
-ngrep
-  url: https://github.com/jpr5/ngrep
-  To run ngrep, open a terminal and type: ngrep -h
-
-scapy
-  url: http://www.secdev.org/projects/scapy/
-  To run scapy, open a terminal and type: scapy
-
-ssldump
-  url: http://www.rtfm.com/ssldump/
-  To run ssldump, open a terminal and type: ssldump -h
-
-sslsplit
-  url: https://github.com/droe/sslsplit
-  To run sslsplit, open a terminal and type: sslsplit -h
-
-tcpdump
-  url: http://www.tcpdump.org
-  To run tcpdump, open a terminal and type: tcpdump -h
-
-tcpflow
-  url: https://github.com/simsong/tcpflow
-  To run tcpflow, open a terminal and type: tcpflow -h
-
-tcpstat
-  url: https://frenchfries.net/paul/tcpstat/
-  To run tcpstat, open a terminal and type: tcpstat -h
-
-tcptrace
-  url: http://www.tcptrace.org
-  To run tcptrace, open a terminal and type: tcptrace -h
-
-tcpxtract
-  url: http://tcpxtract.sourceforge.net/
-  To run tcpxtract, open a terminal and type: tcpxtract -h
-
-whois
-  url: http://www.linux.it/~md/software/
-  To run whois, open a terminal and type: whois -h

salt/common/files/daemon.json

@@ -1,13 +1,11 @@
-{%- set DOCKERRANGE = salt['pillar.get']('docker:range', '172.17.0.0/24') %}
-{%- set DOCKERBIND = salt['pillar.get']('docker:bip', '172.17.0.1/24') %}
 {
   "registry-mirrors": [
     "https://:5000"
   ],
-  "bip": "{{ DOCKERBIND }}",
+  "bip": "172.17.0.1/24",
   "default-address-pools": [
     {
-      "base": "{{ DOCKERRANGE }}",
+      "base": "172.17.0.0/24",
       "size": 24
     }
   ]

salt/common/files/log-rotate.conf

@@ -1,37 +0,0 @@
-{%- set logrotate_conf = salt['pillar.get']('logrotate:conf') %}
-{%- set group_conf = salt['pillar.get']('logrotate:group_conf') %}
-
-
-/opt/so/log/aptcacher-ng/*.log
-/opt/so/log/idstools/*.log
-/opt/so/log/nginx/*.log
-/opt/so/log/soc/*.log
-/opt/so/log/kratos/*.log
-/opt/so/log/kibana/*.log
-/opt/so/log/influxdb/*.log
-/opt/so/log/elastalert/*.log
-/opt/so/log/soctopus/*.log
-/opt/so/log/curator/*.log
-/opt/so/log/fleet/*.log
-/opt/so/log/suricata/*.log
-/opt/so/log/mysql/*.log
-/opt/so/log/telegraf/*.log
-/opt/so/log/redis/*.log
-/opt/so/log/sensoroni/*.log
-/opt/so/log/stenographer/*.log
-/opt/so/log/salt/so-salt-minion-check
-/opt/so/log/salt/minion
-/opt/so/log/salt/master
-/opt/so/log/logscan/*.log
-/nsm/idh/*.log
-{
-    {{ logrotate_conf | indent(width=4) }}
-}
-
-# Playbook's log directory needs additional configuration
-# because Playbook requires a more permissive directory
-/opt/so/log/playbook/*.log
-{
-    {{ logrotate_conf | indent(width=4) }}
-    {{ group_conf | indent(width=4) }}
-}

salt/common/files/sensor-rotate.conf

@@ -1,22 +0,0 @@
-/opt/so/log/sensor_clean.log
-{
-    daily
-    rotate 2
-    missingok
-    nocompress
-    create
-    sharedscripts
-}
-
-/nsm/strelka/log/strelka.log
-{
-    daily
-    rotate 14
-    missingok
-    copytruncate
-    compress
-    create
-    extension .log
-    dateext
-    dateyesterday
-}

salt/common/init.sls

@@ -4,12 +4,16 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
 include:
-  - common.soup_scripts
   - common.packages
 {% if GLOBALS.role in GLOBALS.manager_roles %}
   - manager.elasticsearch # needed for elastic_curl_config state
+  - manager.kibana
 {% endif %}
 
+net.core.wmem_default:
+  sysctl.present:
+    - value: 26214400
+
 # Remove variables.txt from /tmp - This is temp
 rmvariablesfile:
   file.absent:
@@ -49,13 +53,12 @@ so-status.conf:
     - name: /opt/so/conf/so-status/so-status.conf
     - unless: ls /opt/so/conf/so-status/so-status.conf
 
-sosaltstackperms:
+socore_opso_perms:
   file.directory:
-    - name: /opt/so/saltstack
+    - name: /opt/so
     - user: 939
     - group: 939
-    - dir_mode: 770
-
+    
 so_log_perms:
   file.directory:
     - name: /opt/so/log
@@ -112,21 +115,35 @@ elastic_curl_config:
   {% endif %}
 {% endif %}
 
-# Sync some Utilities
-utilsyncscripts:
+
+common_sbin:
   file.recurse:
     - name: /usr/sbin
-    - user: root
-    - group: root
+    - source: salt://common/tools/sbin
+    - user: 939
+    - group: 939
+    - file_mode: 755
+
+common_sbin_jinja:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://common/tools/sbin_jinja
+    - user: 939
+    - group: 939 
     - file_mode: 755
     - template: jinja
-    - source: salt://common/tools/sbin
-    - exclude_pat:
-        - so-common
-        - so-firewall
-        - so-image-common
-        - soup
-        - so-status
+
+{% if not GLOBALS.is_manager%}
+# prior to 2.4.50 these scripts were in common/tools/sbin on the manager because of soup and distributed to non managers
+# these two states remove the scripts from non manager nodes
+remove_soup:
+  file.absent:
+    - name: /usr/sbin/soup
+
+remove_so-firewall:
+  file.absent:
+    - name: /usr/sbin/so-firewall
+{% endif %}
 
 so-status_script:
   file.managed:
@@ -146,56 +163,8 @@ so-sensor-clean:
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
-
-sensorrotatescript:
-  file.managed:
-    - name: /usr/local/bin/sensor-rotate
-    - source: salt://common/cron/sensor-rotate
-    - mode: 755
-
-sensorrotateconf:
-  file.managed:
-    - name: /opt/so/conf/sensor-rotate.conf
-    - source: salt://common/files/sensor-rotate.conf
-    - mode: 644
-
-sensor-rotate:
-  cron.present:
-    - name: /usr/local/bin/sensor-rotate
-    - identifier: sensor-rotate
-    - user: root
-    - minute: '1'
-    - hour: '0'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-
 {% endif %}
 
-commonlogrotatescript:
-  file.managed:
-    - name: /usr/local/bin/common-rotate
-    - source: salt://common/cron/common-rotate
-    - mode: 755
-
-commonlogrotateconf:
-  file.managed:
-    - name: /opt/so/conf/log-rotate.conf
-    - source: salt://common/files/log-rotate.conf
-    - template: jinja
-    - mode: 644
-
-common-rotate:
-  cron.present:
-    - name: /usr/local/bin/common-rotate
-    - identifier: common-rotate
-    - user: root
-    - minute: '1'
-    - hour: '0'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-
 # Create the status directory
 sostatusdir:
   file.directory:
@@ -221,6 +190,14 @@ so-status_check_cron:
     - month: '*'
     - dayweek: '*'
 
+# This cronjob/script runs a check if the node needs restarted, but should be used for future status checks as well
+common_status_check_cron:
+  cron.present:
+    - name: '/usr/sbin/so-common-status-check > /dev/null 2>&1'
+    - identifier: common_status_check
+    - user: root
+    - minute: '*/10'
+
 remove_post_setup_cron:
   cron.absent:
     - name: 'PATH=$PATH:/usr/sbin salt-call state.highstate'
@@ -238,7 +215,7 @@ soversionfile:
 {% endif %}
 
 {% if GLOBALS.so_model and GLOBALS.so_model not in ['SO2AMI01', 'SO2AZI01', 'SO2GCI01'] %}
-  {% if GLOBALS.os == 'Rocky' %}     
+  {% if GLOBALS.os == 'OEL' %}     
 # Install Raid tools
 raidpkgs:
   pkg.installed:
@@ -260,8 +237,7 @@ so-raid-status:
     - month: '*'
     - dayweek: '*'
 
-{% endif %}
-
+  {% endif %}
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/common/packages.sls

@@ -1,6 +1,6 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
-{% if GLOBALS.os == 'Ubuntu' %}
+{% if GLOBALS.os_family == 'Debian' %}
 commonpkgs:
   pkg.installed:
     - skip_suggestions: True
@@ -14,16 +14,24 @@ commonpkgs:
       - software-properties-common
       - apt-transport-https
       - openssl
-      - netcat
+      - netcat-openbsd
       - sqlite3
       - libssl-dev
+      - procps
       - python3-dateutil
+      - python3-docker
       - python3-packaging
-      - python3-watchdog
       - python3-lxml
       - git
+      - rsync
       - vim
+      - tar
+      - unzip
+      {% if grains.oscodename != 'focal' %}
+      - python3-rich
+      {% endif %}
 
+{%     if grains.oscodename == 'focal' %}
 # since Ubuntu requires and internet connection we can use pip to install modules
 python3-pip:
   pkg.installed
@@ -34,34 +42,45 @@ python-rich:
     - target: /usr/local/lib/python3.8/dist-packages/
     - require:
       - pkg: python3-pip
-  
+{%     endif %}
+{% endif %}
+
+{% if GLOBALS.os_family == 'RedHat' %}
+
+remove_mariadb:
+  pkg.removed:
+    - name: mariadb-devel
 
-{% elif GLOBALS.os == 'Rocky' %}     
 commonpkgs:
   pkg.installed:
     - skip_suggestions: True
     - pkgs:
-      - wget
-      - jq
-      - tcpdump
-      - httpd-tools
-      - net-tools
-      - curl
-      - sqlite
-      - mariadb-devel
       - python3-dnf-plugin-versionlock
-      - nmap-ncat
-      - yum-utils
+      - curl
       - device-mapper-persistent-data
-      - lvm2
-      - openssl
+      - fuse
+      - fuse-libs
+      - fuse-overlayfs
+      - fuse-common
+      - fuse3
+      - fuse3-libs
       - git
+      - httpd-tools
+      - jq
+      - lvm2
+      - net-tools
+      - nmap-ncat
+      - procps-ng
       - python3-docker
       - python3-m2crypto
-      - rsync
-      - python3-rich
-      - python3-pyyaml
-      - python3-watchdog
       - python3-packaging
+      - python3-pyyaml
+      - python3-rich
+      - rsync
+      - sqlite
+      - tcpdump
       - unzip
+      - wget
+      - yum-utils
+
 {% endif %}

salt/common/soup_scripts.sls

@@ -1,13 +1,117 @@
-# Sync some Utilities
-soup_scripts:
-  file.recurse:
-    - name: /usr/sbin
-    - user: root
-    - group: root
-    - file_mode: 755
-    - source: salt://common/tools/sbin
-    - include_pat:
-        - so-common
-        - so-firewall
-        - so-image-common
-        - soup
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% if '2.4' in salt['cp.get_file_str']('/etc/soversion') %}
+
+{%   import_yaml '/opt/so/saltstack/local/pillar/global/soc_global.sls' as SOC_GLOBAL %}
+{%   if SOC_GLOBAL.global.airgap %}
+{%     set UPDATE_DIR='/tmp/soagupdate/SecurityOnion' %}
+{%   else %}
+{%     set UPDATE_DIR='/tmp/sogh/securityonion' %}
+{%   endif %}
+
+remove_common_soup:
+  file.absent:
+    - name: /opt/so/saltstack/default/salt/common/tools/sbin/soup
+
+remove_common_so-firewall:
+  file.absent:
+    - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-firewall
+
+# This section is used to put the scripts in place in the Salt file system
+# in case a state run tries to overwrite what we do in the next section.
+copy_so-common_common_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-common
+    - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common
+    - force: True
+    - preserve: True
+
+copy_so-image-common_common_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common
+    - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common
+    - force: True
+    - preserve: True
+
+copy_soup_manager_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/soup
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup
+    - force: True
+    - preserve: True
+
+copy_so-firewall_manager_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-firewall
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
+    - force: True
+    - preserve: True
+
+copy_so-yaml_manager_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-yaml.py
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
+    - force: True
+    - preserve: True
+
+copy_so-repo-sync_manager_tools_sbin:
+  file.copy:
+    - name: /opt/so/saltstack/default/salt/manager/tools/sbin/so-repo-sync
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
+    - preserve: True
+
+# This section is used to put the new script in place so that it can be called during soup.
+# It is faster than calling the states that normally manage them to put them in place.
+copy_so-common_sbin:
+  file.copy:
+    - name: /usr/sbin/so-common
+    - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-common
+    - force: True
+    - preserve: True
+
+copy_so-image-common_sbin:
+  file.copy:
+    - name: /usr/sbin/so-image-common
+    - source: {{UPDATE_DIR}}/salt/common/tools/sbin/so-image-common
+    - force: True
+    - preserve: True
+
+copy_soup_sbin:
+  file.copy:
+    - name: /usr/sbin/soup
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/soup
+    - force: True
+    - preserve: True
+
+copy_so-firewall_sbin:
+  file.copy:
+    - name: /usr/sbin/so-firewall
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-firewall
+    - force: True
+    - preserve: True
+
+copy_so-yaml_sbin:
+  file.copy:
+    - name: /usr/sbin/so-yaml.py
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-yaml.py
+    - force: True
+    - preserve: True
+
+copy_so-repo-sync_sbin:
+  file.copy:
+    - name: /usr/sbin/so-repo-sync
+    - source: {{UPDATE_DIR}}/salt/manager/tools/sbin/so-repo-sync
+    - force: True
+    - preserve: True
+
+{% else %}
+fix_23_soup_sbin:
+  cmd.run:
+    - name: curl -s -f -o /usr/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
+fix_23_soup_salt:
+  cmd.run:
+    - name: curl -s -f -o /opt/so/saltstack/defalt/salt/common/tools/sbin/soup https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.3/main/salt/common/tools/sbin/soup
+{% endif %}

salt/common/tools/sbin/so-checkin

@@ -5,8 +5,13 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
-
 . /usr/sbin/so-common
 
-salt-call state.highstate -l info 
+cat << EOF
+
+so-checkin will run a full salt highstate to apply all salt states. If a highstate is already running, this request will be queued and so it may pause for a few minutes before you see any more output. For more information about so-checkin and salt, please see:
+https://docs.securityonion.net/en/2.4/salt.html
+
+EOF
+
+salt-call state.highstate -l info queue=True

salt/common/tools/sbin/so-common

@@ -5,6 +5,16 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+# Elastic agent is not managed by salt. Because of this we must store this base information in a 
+# script that accompanies the soup system. Since so-common is one of those special soup files, 
+# and since this same logic is required during installation, it's included in this file.
+ELASTIC_AGENT_TARBALL_VERSION="8.10.4"
+ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
+
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
 
@@ -21,6 +31,11 @@ if ! echo "$PATH" | grep -q "/usr/sbin"; then
   export PATH="$PATH:/usr/sbin"
 fi
 
+# See if a proxy is set. If so use it.
+if [ -f /etc/profile.d/so-proxy.sh ]; then
+  . /etc/profile.d/so-proxy.sh
+fi
+
 # Define a banner to separate sections
 banner="========================================================================="
 
@@ -123,76 +138,99 @@ check_elastic_license() {
 }
 
 check_salt_master_status() {
-	local timeout=$1
-	echo "Checking if we can talk to the salt master"
-	salt-call state.show_top concurrent=true
-
-	return
+	local count=0
+    local attempts="${1:- 10}"
+	current_time="$(date '+%b %d %H:%M:%S')"
+	echo "Checking if we can access the salt master and that it is ready at: ${current_time}"
+    while ! salt-call state.show_top -l error concurrent=true 1> /dev/null; do
+        current_time="$(date '+%b %d %H:%M:%S')"
+        echo "Can't access salt master or it is not ready at: ${current_time}"
+        ((count+=1))
+        if [[ $count -eq $attempts ]]; then
+			# 10 attempts takes about 5.5 minutes
+            echo "Gave up trying to access salt-master"
+            return 1
+        fi
+    done
+	current_time="$(date '+%b %d %H:%M:%S')"
+	echo "Successfully accessed and salt master ready at: ${current_time}"
+    return 0
 }
 
+# this is only intended to be used to check the status of the minion from a salt master
 check_salt_minion_status() {
-	local timeout=$1
-	echo "Checking if the salt minion will respond to jobs" >> "$setup_log" 2>&1
-	salt "$MINION_ID" test.ping -t $timeout > /dev/null 2>&1
+	local minion="$1"
+	local timeout="${2:-5}"
+	local logfile="${3:-'/dev/stdout'}"
+	echo "Checking if the salt minion: $minion will respond to jobs" >> "$logfile" 2>&1
+	salt "$minion" test.ping -t $timeout > /dev/null 2>&1
 	local status=$?
 	if [ $status -gt 0 ]; then
-		echo "  Minion did not respond" >> "$setup_log" 2>&1
+		echo "  Minion did not respond" >> "$logfile" 2>&1
 	else
-		echo "  Received job response from salt minion" >> "$setup_log" 2>&1
+		echo "  Received job response from salt minion" >> "$logfile" 2>&1
 	fi
 
 	return $status
 }
 
-
-
 copy_new_files() {
   # Copy new files over to the salt dir
   cd $UPDATE_DIR
-  rsync -a salt $DEFAULT_SALT_DIR/
-  rsync -a pillar $DEFAULT_SALT_DIR/
+  rsync -a salt $DEFAULT_SALT_DIR/ --delete
+  rsync -a pillar $DEFAULT_SALT_DIR/ --delete
   chown -R socore:socore $DEFAULT_SALT_DIR/
   chmod 755 $DEFAULT_SALT_DIR/pillar/firewall/addfirewall.sh
   cd /tmp
 }
 
+create_local_directories() {
+	echo "Creating local pillar and salt directories if needed"
+	PILLARSALTDIR=$1
+	local_salt_dir="/opt/so/saltstack/local"
+	for i in "pillar" "salt"; do
+		for d in $(find $PILLARSALTDIR/$i -type d); do
+			suffixdir=${d//$PILLARSALTDIR/}
+			if [ ! -d "$local_salt_dir/$suffixdir" ]; then
+				mkdir -pv $local_salt_dir$suffixdir
+			fi
+		done
+		chown -R socore:socore $local_salt_dir/$i
+	done
+}
+
 disable_fastestmirror() {
 	sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
 }
 
-elastic_fleet_integration_create() {
+download_and_verify() {
+	source_url=$1
+	source_md5_url=$2
+	dest_file=$3
+	md5_file=$4
+	expand_dir=$5
 
-    JSON_STRING=$1
+	if [[ -n "$expand_dir" ]]; then
+		mkdir -p "$expand_dir"
+	fi
 
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
+	if ! verify_md5_checksum "$dest_file" "$md5_file"; then
+		retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L '$source_url' --output '$dest_file'" "" ""
+		retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L '$source_md5_url' --output '$md5_file'"  "" ""
+
+		if verify_md5_checksum "$dest_file" "$md5_file"; then
+		  echo "Source file and checksum are good."
+		else
+		  echo "Unable to download and verify the source file and checksum."
+		  return 1
+		fi
+	fi
+
+	if [[ -n "$expand_dir" ]]; then
+		tar -xf "$dest_file" -C "$expand_dir"
+	fi
 }
 
-elastic_fleet_policy_create() {
-
-    NAME=$1
-    DESC=$2
-    FLEETSERVER=$3
-
-    JSON_STRING=$( jq -n \
-                    --arg NAME "$NAME" \
-                    --arg DESC "$DESC" \
-                    --arg FLEETSERVER "$FLEETSERVER" \
-                    '{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":1209600,"has_fleet_server":$FLEETSERVER}'
-                    )
-    # Create Fleet Policy
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" 
-
-}
-
-elastic_fleet_policy_update() {
-
-    POLICYID=$1
-    JSON_STRING=$2
-
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/$POLICYID" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-}
-
-
 elastic_license() {
 
 read -r -d '' message <<- EOM
@@ -230,20 +268,29 @@ get_random_value() {
 	head -c 5000 /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w $length | head -n 1
 }
 
-gpg_rpm_import() {
-	if [[ "$OS" == "rocky" ]]; then
-	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
-	        local RPMKEYSLOC="../salt/repo/client/files/rocky/keys"
-	    else
-	        local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/rocky/keys"
-	    fi
-	
-	    RPMKEYS=('RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
+get_agent_count() {
+	if [ -f /opt/so/log/agents/agentstatus.log ]; then
+		AGENTCOUNT=$(cat /opt/so/log/agents/agentstatus.log | grep -wF active | awk '{print $2}')
+	else
+		AGENTCOUNT=0
+	fi
+}
 
-	    for RPMKEY in "${RPMKEYS[@]}"; do
+gpg_rpm_import() {
+	if [[ $is_oracle ]]; then
+	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
+	        local RPMKEYSLOC="../salt/repo/client/files/$OS/keys"
+	    else
+	        local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys"
+	    fi
+		RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub')
+		for RPMKEY in "${RPMKEYS[@]}"; do
     	        rpm --import $RPMKEYSLOC/$RPMKEY
 	        echo "Imported $RPMKEY"
 	    done
+	elif [[ $is_rpm ]]; then
+		echo "Importing the security onion GPG key"
+		rpm --import ../salt/repo/client/files/oracle/keys/securityonion.pub
 	fi
 }
 
@@ -256,12 +303,15 @@ init_monitor() {
 	
 	if [[ $MONITORNIC == "bond0" ]]; then 
 	  BIFACES=$(lookup_bond_interfaces)
+	  for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload generic-receive-offload generic-segmentation-offload tcp-segmentation-offload; do
+	    ethtool -K "$MONITORNIC" "$i" off;
+  	  done
 	else
 	  BIFACES=$MONITORNIC
 	fi
 
 	for DEVICE_IFACE in $BIFACES; do
-	  for i in rx tx sg tso ufo gso gro lro; do
+	  for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload generic-receive-offload generic-segmentation-offload tcp-segmentation-offload; do
 	    ethtool -K "$DEVICE_IFACE" "$i" off;
   	  done
 	  ip link set dev "$DEVICE_IFACE" arp off multicast off allmulticast off promisc on
@@ -275,7 +325,7 @@ is_manager_node() {
 is_sensor_node() {
 	# Check to see if this is a sensor (forward) node
 	is_single_node_grid && return 0
-	grep "role: so-" /etc/salt/grains | grep -E "sensor|heavynode|helix" &> /dev/null
+	grep "role: so-" /etc/salt/grains | grep -E "sensor|heavynode" &> /dev/null
 }
 
 is_single_node_grid() {
@@ -307,7 +357,7 @@ lookup_salt_value() {
 		local=""
 	fi
 
-	salt-call --no-color  ${kind}.get ${group}${key} --out=${output} ${local}
+	salt-call -lerror --no-color  ${kind}.get ${group}${key} --out=${output} ${local}
 }
 
 lookup_pillar() {
@@ -333,6 +383,24 @@ lookup_role() {
 	echo ${pieces[1]}
 }
 
+is_feature_enabled() {
+	feature=$1
+	enabled=$(lookup_salt_value features)
+	for cur in $enabled; do
+		if [[ "$feature" == "$cur" ]]; then
+			return 0
+		fi
+	done
+	return 1
+}
+
+read_feat() {
+	if [ -f /opt/so/log/sostatus/lks_enabled ]; then
+		lic_id=$(cat /opt/so/saltstack/local/pillar/soc/license.sls | grep license_id: | awk '{print $2}')
+		echo "$lic_id/$(cat /opt/so/log/sostatus/lks_enabled)/$(cat /opt/so/log/sostatus/fps_enabled)"
+	fi
+}
+
 require_manager() {
 	if is_manager_node; then
 		echo "This is a manager, so we can proceed."
@@ -364,6 +432,10 @@ retry() {
 								echo "<Start of output>"
 								echo "$output"
 								echo "<End of output>"
+								if [[ $exitcode -eq 0 ]]; then
+									echo "Forcing exit code to 1"
+									exitcode=1
+								fi
                         fi
                 elif [ -n "$failedOutput" ]; then
                         if [[ "$output" =~ "$failedOutput" ]]; then
@@ -372,7 +444,7 @@ retry() {
 								echo "$output"
 								echo "<End of output>"
 								if [[ $exitcode -eq 0 ]]; then
-									echo "The exitcode was 0, but we are setting to 1 since we found $failedOutput in the output."
+									echo "Forcing exit code to 1"
 									exitcode=1
 								fi
                         else
@@ -410,25 +482,82 @@ run_check_net_err() {
 	fi
 }
 
+wait_for_salt_minion() {
+	local minion="$1"
+	local timeout="${2:-5}"
+	local logfile="${3:-'/dev/stdout'}"
+	retry 60 5 "journalctl -u salt-minion.service | grep 'Minion is ready to receive requests'" >> "$logfile" 2>&1 || fail
+	local attempt=0
+	# each attempts would take about 15 seconds
+	local maxAttempts=20
+	until check_salt_minion_status "$minion" "$timeout" "$logfile"; do
+		attempt=$((attempt+1))
+		if [[ $attempt -eq $maxAttempts ]]; then
+			return 1
+		fi
+		sleep 10
+	done
+	return 0
+}
+
 salt_minion_count() {
 	local MINIONDIR="/opt/so/saltstack/local/pillar/minions"
 	MINIONCOUNT=$(ls -la $MINIONDIR/*.sls | grep -v adv_ | wc -l)
 
 }
 
-set_cron_service_name() {
-	if [[ "$OS" == "rocky" ]]; then
-		cron_service_name="crond"
-	else
-		cron_service_name="cron"
-	fi
-}
-
 set_os() {
 	if [ -f /etc/redhat-release ]; then
-		OS=rocky
-	else
-		OS=ubuntu
+		if grep -q "Rocky Linux release 9" /etc/redhat-release; then
+			OS=rocky
+			OSVER=9
+			is_rocky=true
+			is_rpm=true
+		elif grep -q "CentOS Stream release 9" /etc/redhat-release; then
+			OS=centos
+			OSVER=9
+			is_centos=true
+			is_rpm=true
+		elif grep -q "AlmaLinux release 9" /etc/redhat-release; then
+			OS=alma
+			OSVER=9
+			is_alma=true
+			is_rpm=true
+		elif grep -q "Red Hat Enterprise Linux release 9" /etc/redhat-release; then
+			if [ -f /etc/oracle-release ]; then
+				OS=oracle
+				OSVER=9
+				is_oracle=true
+				is_rpm=true
+			else
+				OS=rhel
+				OSVER=9
+				is_rhel=true
+				is_rpm=true
+			fi
+		fi
+		cron_service_name="crond"
+	elif [ -f /etc/os-release ]; then
+		if grep -q "UBUNTU_CODENAME=focal" /etc/os-release; then
+			OSVER=focal
+			UBVER=20.04
+			OS=ubuntu
+			is_ubuntu=true
+			is_deb=true
+		elif grep -q "UBUNTU_CODENAME=jammy" /etc/os-release; then
+			OSVER=jammy
+			UBVER=22.04
+			OS=ubuntu
+			is_ubuntu=true
+			is_deb=true
+		elif grep -q "VERSION_CODENAME=bookworm" /etc/os-release; then
+			OSVER=bookworm
+			DEBVER=12
+			is_debian=true
+			OS=debian
+			is_deb=true
+		fi
+		cron_service_name="cron"
 	fi
 }
 
@@ -437,7 +566,7 @@ set_minionid() {
 }
 
 set_palette() {
-	if [ "$OS" == ubuntu ]; then
+	if [[ $is_deb ]]; then
 	    update-alternatives --set newt-palette /etc/newt/palette.original
     fi
 }
@@ -461,6 +590,19 @@ set_version() {
 	fi
 }
 
+status () {
+    printf "\n=========================================================================\n$(date) | $1\n=========================================================================\n"
+}
+
+sync_options() {
+	set_version
+	set_os
+	salt_minion_count
+	get_agent_count
+	
+	echo "$VERSION/$OS/$(uname -r)/$MINIONCOUNT:$AGENTCOUNT/$(read_feat)"
+}
+
 systemctl_func() {
 	local action=$1
 	local echo_action=$1
@@ -484,6 +626,11 @@ has_uppercase() {
 		|| return 1
 }
 
+update_elastic_agent() {
+	echo "Checking if Elastic Agent update is necessary..."
+	download_and_verify "$ELASTIC_AGENT_URL" "$ELASTIC_AGENT_MD5_URL" "$ELASTIC_AGENT_FILE" "$ELASTIC_AGENT_MD5" "$ELASTIC_AGENT_EXPANSION_DIR"
+}
+
 valid_cidr() {
 	# Verify there is a backslash in the string
 	echo "$1" | grep -qP "^[^/]+/[^/]+$" || return 1
@@ -637,6 +784,23 @@ valid_username() {
 	echo "$user" | grep -qP '^[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}\$)$' && return 0 || return 1
 }
 
+verify_md5_checksum() {
+	data_file=$1
+	md5_file=${2:-${data_file}.md5}
+
+	if [[ ! -f "$dest_file" || ! -f "$md5_file" ]]; then
+		return 2
+	fi
+
+	SOURCEHASH=$(md5sum "$data_file" | awk '{ print $1 }')
+	HASH=$(cat "$md5_file")
+
+	if [[ "$HASH" == "$SOURCEHASH" ]]; then
+		return 0
+	fi
+	return 1
+}
+
 wait_for_web_response() {
 	url=$1
 	expected=$2

salt/common/tools/sbin/so-common-status-check

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+import sys
+import subprocess
+import os
+import json
+
+sys.path.append('/opt/saltstack/salt/lib/python3.10/site-packages/')
+import salt.config
+import salt.loader
+
+__opts__ = salt.config.minion_config('/etc/salt/minion')
+__grains__ = salt.loader.grains(__opts__)
+
+def check_needs_restarted():
+  osfam = __grains__['os_family']
+  val = '0'
+  outfile = "/opt/so/log/sostatus/needs_restarted"
+
+  if osfam == 'Debian':
+    if os.path.exists('/var/run/reboot-required'):
+      val = '1'
+  elif osfam == 'RedHat':
+    cmd = 'needs-restarting -r > /dev/null 2>&1'
+    try:
+      needs_restarting = subprocess.check_call(cmd, shell=True)
+    except subprocess.CalledProcessError:
+      val = '1'
+  else:
+    fail("Unsupported OS")
+
+  with open(outfile, 'w') as f:
+    f.write(val)
+
+def check_for_fps():
+    feat = 'fps'
+    feat_full = feat.replace('ps', 'ips')
+    fps = 0
+    try:
+        result = subprocess.run([feat_full + '-mode-setup', '--is-enabled'], stdout=subprocess.PIPE)
+        if result.returncode == 0:
+            fps = 1
+    except FileNotFoundError:
+        fn = '/proc/sys/crypto/' + feat_full + '_enabled'
+        try:
+          with open(fn, 'r') as f:
+              contents = f.read()
+              if '1' in contents:
+                  fps = 1
+        except:
+          # Unknown, so assume 0
+          fps = 0
+
+    with open('/opt/so/log/sostatus/fps_enabled', 'w') as f:
+       f.write(str(fps))
+
+def check_for_lks():
+    feat = 'Lks'
+    feat_full = feat.replace('ks', 'uks')
+    lks = 0
+    result = subprocess.run(['lsblk', '-p', '-J'], check=True, stdout=subprocess.PIPE)
+    data = json.loads(result.stdout)
+    for device in data['blockdevices']:
+        if 'children' in device:
+            for gc in device['children']:
+                if 'children' in gc:
+                    try:
+                        arg = 'is' + feat_full
+                        result = subprocess.run(['cryptsetup', arg, gc['name']], stdout=subprocess.PIPE)
+                        if result.returncode == 0:
+                           lks = 1
+                    except FileNotFoundError:
+                        for ggc in gc['children']:
+                            if 'crypt' in ggc['type']:
+                                lks = 1
+        if lks:
+            break
+    with open('/opt/so/log/sostatus/lks_enabled', 'w') as f:
+       f.write(str(lks))
+
+def fail(msg):
+  print(msg, file=sys.stderr)
+  sys.exit(1)
+
+def main():
+  proc = subprocess.run(['id', '-u'], stdout=subprocess.PIPE, encoding="utf-8")
+  if proc.stdout.strip() != "0":
+    fail("This program must be run as root")
+  # Ensure that umask is 0022 so that files created by this script have rw-r-r permissions
+  org_umask = os.umask(0o022)
+  check_needs_restarted()
+  check_for_fps()
+  check_for_lks()
+  # Restore umask to whatever value was set before this script was run. SXIG sets to 0077 rw---
+  os.umask(org_umask)
+
+if __name__ == "__main__":
+  main()

salt/common/tools/sbin/so-elastic-agent-gen-installers

@@ -1,34 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
-# this file except in compliance with the Elastic License 2.0.
-
-#so-elastic-agent-gen-installers $FleetHost $EnrollmentToken 
-
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
-. /usr/sbin/so-common
-
-ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints")) | .api_key')
-
-#FLEETHOST=$(lookup_pillar "server:url" "elasticfleet")
-FLEETHOST="{{ GLOBALS.manager_ip }}"
-
-#FLEETHOST=$1
-#ENROLLMENTOKEN=$2
-CONTAINERGOOS=( "linux" "darwin" "windows" )
-
-#rm -rf /tmp/elastic-agent-workspace
-#mkdir -p /tmp/elastic-agent-workspace
-
-for OS in "${CONTAINERGOOS[@]}"
-do
-    printf "\n\nGenerating $OS Installer..."
-    #cp /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/so-elastic-agent-*-$OS-x86_64.tar.gz /tmp/elastic-agent-workspace/$OS.tar.gz
-    docker run -e CGO_ENABLED=0 -e GOOS=$OS \
-    --mount type=bind,source=/etc/ssl/certs/,target=/workspace/files/cert/ \
-    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
-    {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHost=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_$OS
-    printf "\n $OS Installer Generated..."
-done

salt/common/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -1,21 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-# Initial Endpoints
-for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/endpoints-initial/*.json
-do
-  printf "\n\nInitial Endpoint Policy - Loading $INTEGRATION\n"
-  elastic_fleet_integration_create "@$INTEGRATION"
-done
-
-# Grid Nodes
-for INTEGRATION in /opt/so/saltstack/default/salt/elasticfleet/files/integrations/grid-nodes/*.json
-do
-  printf "\n\nGrid Nodes Policy - Loading $INTEGRATION\n"
-  elastic_fleet_integration_create "@$INTEGRATION"
-done

salt/common/tools/sbin/so-elastic-fleet-setup

@@ -1,92 +0,0 @@
-
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
-# this file except in compliance with the Elastic License 2.0.
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
-. /usr/sbin/so-common
-
-printf "\n### Create ES Token ###\n"
-ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' |  jq -r .value)
-
-### Create Outputs & Fleet URLs ###
-printf "\nAdd Manager Elasticsearch Ouput...\n"
-ESCACRT=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
-JSON_STRING=$( jq -n \
-                  --arg ESCACRT "$ESCACRT" \
-                  '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200","https://{{ GLOBALS.manager }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' )
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-printf "\n\n"
-
-printf "\nCreate Logstash Output if node is not an Import or Eval install\n"
-{% if grains.role not in ['so-import', 'so-eval'] %}
-LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt)
-LOGSTASHKEY=$(openssl rsa -in  /etc/pki/elasticfleet-logstash.key)
-LOGSTASHCA=$(openssl x509 -in  /etc/pki/tls/certs/intca.crt)
-JSON_STRING=$( jq -n \
-                  --arg LOGSTASHCRT "$LOGSTASHCRT" \
-                  --arg LOGSTASHKEY "$LOGSTASHKEY" \
-                  --arg LOGSTASHCA "$LOGSTASHCA" \
-                  '{"name":"grid-logstash","is_default":true,"is_default_monitoring":true,"id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055", "{{ GLOBALS.manager }}:5055"],"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]},"proxy_id":null}'
-                  )
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-printf "\n\n"
-{%- endif %}
-
-printf "\nAdd SO-Manager Fleet URL\n"
-## This array replaces whatever URLs are currently configured
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d  '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220", "https://{{ GLOBALS.manager }}:8220"]}'
-printf "\n\n"
-
-
-### Create Policies & Associated Integration Configuration ###
-
-# Manager Fleet Server Host
-elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" | jq
-
-#Temp Fixup for ES Output bug
-JSON_STRING=$( jq -n \
-                --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \
-                '{"name": $NAME,"description": $NAME,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":120,"data_output_id":"so-manager_elasticsearch"}'
-                )
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_{{ GLOBALS.hostname }}" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-
-# Initial Endpoints Policy
-elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false"
-
-# Grid Nodes Policy
-elastic_fleet_policy_create "so-grid-nodes" "SO Grid Node Policy" "false"
-
-# Load Integrations for default policies
-so-elastic-fleet-integration-policy-load
-
-### Finalization ###
-
-# Query for Enrollment Tokens for default policies
-ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-default")) | .api_key')
-GRIDNODESENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes")) | .api_key')
-
-# Store needed data in minion pillar
-pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls
-printf '%s\n'\
-    "elasticfleet:"\
-    "  server:"\
-    "    es_token: '$ESTOKEN'"\
-    "    endpoints_enrollment: '$ENDPOINTSENROLLMENTOKEN'"\
-    "    grid_enrollment: '$GRIDNODESENROLLMENTOKEN'"\
-    "" >> "$pillar_file"
-
-#Store Grid Nodes Enrollment token in Global pillar
-global_pillar_file=/opt/so/saltstack/local/pillar/global/soc_global.sls
-printf '%s\n'\
-    "  fleet_grid_enrollment_token: '$GRIDNODESENROLLMENTOKEN'"\
-    "" >> "$global_pillar_file"
-
-# Call Elastic-Fleet Salt State
-salt-call state.apply elasticfleet queue=True
-
-# Generate installers & install Elastic Agent on the node
-so-elastic-agent-gen-installers
-salt-call state.apply elasticfleet.install_agent_grid queue=True

salt/common/tools/sbin/so-elastic-restart

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%}
-/usr/sbin/so-restart elasticsearch $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
-/usr/sbin/so-restart kibana $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-restart logstash $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-restart curator $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
-/usr/sbin/so-restart elastalert $1
-{%- endif %}

salt/common/tools/sbin/so-elastic-start

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%}
-/usr/sbin/so-start elasticsearch $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
-/usr/sbin/so-start kibana $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-start logstash $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-start curator $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
-/usr/sbin/so-start elastalert $1
-{%- endif %}

salt/common/tools/sbin/so-elastic-stop

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%}
-/usr/sbin/so-stop elasticsearch $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
-/usr/sbin/so-stop kibana $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-stop logstash $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-stop curator $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
-/usr/sbin/so-stop elastalert $1
-{%- endif %}

salt/common/tools/sbin/so-elasticsearch-ilm-policy-load

@@ -1,21 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
-{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %}
-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
-
-{%- for index, settings in ES_INDEX_SETTINGS.items() %}
-  {%- if settings.policy is defined %}
-echo
-echo "Setting up {{ index }}-logs policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
-echo
-  {%- endif %}
-{%- endfor %}
-echo

salt/common/tools/sbin/so-elasticsearch-indices-rw

@@ -1,15 +0,0 @@
-#!/bin/bash
-#
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-IP={{ salt['grains.get']('ip_interfaces').get(salt['pillar.get']('sensor:mainint', salt['pillar.get']('manager:mainint', salt['pillar.get']('elasticsearch:mainint', salt['pillar.get']('host:mainint')))))[0] }}
-ESPORT=9200
-
-echo "Removing read only attributes for indices..."
-echo
-curl -K /opt/so/conf/elasticsearch/curl.config -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;

salt/common/tools/sbin/so-firewall

@@ -1,104 +0,0 @@
-#!/usr/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-if [[ $# -lt 1 ]]; then
-  echo "Usage: $0 --role=<ROLE> --ip=<IP ADDRESS> --apply=<true|false>"
-  echo ""
-  echo " Example: so-firewall --role=sensor --ip=192.168.254.100 --apply=true"
-  echo ""
-  exit 1
-fi
-
-for i in "$@"; do
-	case $i in
-		-r=*|--role=*)
-			ROLE="${i#*=}"
-			shift 
-			;;
-		-i=*|--ip=*)
-			IP="${i#*=}"
-			shift 
-			;;
-		-a=*|--apply*)
-			APPLY="${i#*=}"
-			shift
-			;;
-		-*|--*)
-			echo "Unknown option $i"
-			exit 1
-			;;
-		*)
-		;;
-	esac
-done
-
-ROLE=${ROLE,,}
-APPLY=${APPLY,,}
-
-function rolecall() {
-	THEROLE=$1
-	THEROLES="analyst analyst_workstations beats_endpoint beats_endpoint_ssl elastic_agent_endpoint elasticsearch_rest endgame eval fleet heavynodes idh manager managersearch receivers searchnodes sensors standalone strelka_frontend syslog"
-
-	for AROLE in $THEROLES; do
-		if [ "$AROLE" = "$THEROLE" ]; then
-			return 0
-		fi
-	done
-	return 1
-}
-
-# Make sure the required options are specified
-if [ -z "$ROLE" ]; then
-	echo "Please specify a role with --role="
-	exit 1
-fi
-if [ -z "$IP" ]; then
-	echo "Please specify an IP address with --ip="
-	exit 1
-fi
-
-# Are we dealing with a role that this script supports?
-if rolecall "$ROLE"; then
-	echo "$ROLE is a supported role"
-else
-	echo "This is not a supported role"
-	exit 1
-fi
-
- # Are we dealing with an IP?
-if verify_ip4 "$IP"; then
-	echo "$IP is a valid IP or CIDR"
-else
-	echo "$IP is not a valid IP or CIDR"
-	exit 1
-fi
-
-local_salt_dir=/opt/so/saltstack/local/salt/firewall
-
-# Let's see if the file exists and if it does, let's see if the IP exists.
-if [ -f "$local_salt_dir/hostgroups/$ROLE" ]; then
-	if grep -q $IP "$local_salt_dir/hostgroups/$ROLE"; then
-		echo "Host already exists"
-		exit 0
-	fi
-fi
-
-# If you have reached this part of your quest then let's add the IP
-echo "Adding $IP to the $ROLE role"
-echo "$IP" >> $local_salt_dir/hostgroups/$ROLE
-
-# Check to see if we are applying this right away.
-if [ "$APPLY" = "true" ]; then
-	echo "Applying the firewall rules"
-	salt-call state.apply firewall queue=True
-	echo "Firewall rules have been applied... Review logs further if there were errors."
-	echo ""
-else
-	echo "Firewall rules will be applied next salt run"
-fi

salt/common/tools/sbin/so-helix-apikey

@@ -1,27 +0,0 @@
-#!/bin/bash
-
-local_salt_dir=/opt/so/saltstack/local
-
-got_root() {
-
-  # Make sure you are root
-  if [ "$(id -u)" -ne 0 ]; then
-          echo "This script must be run using sudo!"
-          exit 1
-  fi
-
-}
-
-got_root
-if [ ! -f $local_salt_dir/pillar/fireeye/init.sls ]; then
-  echo "This is nto configured for Helix Mode. Please re-install."
-  exit
-else
-  echo "Enter your Helix API Key: "
-  read APIKEY
-  sed -i "s/^    api_key.*/    api_key: $APIKEY/g" $local_salt_dir/pillar/fireeye/init.sls
-  docker stop so-logstash
-  docker rm so-logstash
-  echo "Restarting Logstash for updated key"
-  salt-call state.apply logstash queue=True
-fi

salt/common/tools/sbin/so-image-common

@@ -38,27 +38,26 @@ container_list() {
       "so-zeek"
       "so-elastic-agent"
       "so-elastic-agent-builder"
+      "so-elastic-fleet-package-registry"
     )
   elif [ $MANAGERCHECK != 'so-helix' ]; then
     TRUSTED_CONTAINERS=(
-      "so-curator"
       "so-elastalert"
       "so-elastic-agent"
       "so-elastic-agent-builder"
+      "so-elastic-fleet-package-registry"
       "so-elasticsearch"
       "so-idh"
       "so-idstools"
       "so-influxdb"
+      "so-kafka"
       "so-kibana"
       "so-kratos"
       "so-logstash"
-      "so-mysql"
       "so-nginx"
       "so-pcaptools"
-      "so-playbook"
       "so-redis"
       "so-soc"
-      "so-soctopus"
       "so-steno"
       "so-strelka-backend"
       "so-strelka-filestream"
@@ -66,7 +65,7 @@ container_list() {
       "so-strelka-manager"
       "so-suricata"
       "so-telegraf"
-      "so-zeek" 
+      "so-zeek"
     )
   else
     TRUSTED_CONTAINERS=(
@@ -135,7 +134,7 @@ update_docker_containers() {
   for i in "${TRUSTED_CONTAINERS[@]}"
   do
     if [ -z "$PROGRESS_CALLBACK" ]; then
-      echo "Downloading $i" >> "$LOG_FILE" 2>&1 
+      echo "Downloading $i" >> "$LOG_FILE" 2>&1
     else
       $PROGRESS_CALLBACK $i
     fi

salt/common/tools/sbin/so-import-evtx

@@ -1,155 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set MANAGER = salt['grains.get']('master') %}
-{%- set VERSION = salt['pillar.get']('global:soversion') %}
-{%- set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
-{%- set MANAGERIP = salt['pillar.get']('global:managerip') %}
-{%- set URLBASE = salt['pillar.get']('global:url_base') %}
-{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
-{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
-
-INDEX_DATE=$(date +'%Y.%m.%d')
-RUNID=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1)
-LOG_FILE=/nsm/import/evtx-import.log
-
-. /usr/sbin/so-common
-
-function usage {
-  cat << EOF
-Usage: $0 <evtx-file-1> [evtx-file-2] [evtx-file-*]
-
-Imports one or more evtx files into Security Onion. The evtx files will be analyzed and made available for review in the Security Onion toolset.
-EOF
-}
-
-
-function evtx2es() {
-  EVTX=$1
-  HASH=$2
-  
-  docker run --rm \
-    -v "$EVTX:/tmp/data.evtx" \
-    -v "/nsm/import/$HASH/evtx/:/tmp/evtx/" \
-    -v "/nsm/import/evtx-end_newest:/tmp/newest" \
-    -v "/nsm/import/evtx-start_oldest:/tmp/oldest" \
-    --entrypoint "/evtx_calc_timestamps.sh" \
-    {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} >> $LOG_FILE 2>&1
-}
-
-# if no parameters supplied, display usage
-if [ $# -eq 0 ]; then
-  usage
-  exit 1
-fi
-
-# ensure this is a Manager node
-require_manager
-
-# verify that all parameters are files
-for i in "$@"; do
-  if ! [ -f "$i" ]; then
-    usage
-    echo "\"$i\" is not a valid file!"
-    exit 2
-  fi
-done
-
-# track if we have any valid or invalid evtx
-INVALID_EVTXS="no"
-VALID_EVTXS="no"
-
-# track oldest start and newest end so that we can generate the Kibana search hyperlink at the end
-START_OLDEST="2050-12-31"
-END_NEWEST="1971-01-01"
-
-touch /nsm/import/evtx-start_oldest
-touch /nsm/import/evtx-end_newest
-
-echo $START_OLDEST > /nsm/import/evtx-start_oldest
-echo $END_NEWEST > /nsm/import/evtx-end_newest
-
-# paths must be quoted in case they include spaces
-for EVTX in "$@"; do
-  EVTX=$(/usr/bin/realpath "$EVTX")
-  echo "Processing Import: ${EVTX}"
-
-  # generate a unique hash to assist with dedupe checks
-  HASH=$(md5sum "${EVTX}" | awk '{ print $1 }')
-  HASH_DIR=/nsm/import/${HASH}
-  echo "- assigning unique identifier to import: $HASH"
-
-  if [ -d $HASH_DIR ]; then
-    echo "- this EVTX has already been imported; skipping"
-    INVALID_EVTXS="yes"
-  else
-    VALID_EVTXS="yes"
-
-    EVTX_DIR=$HASH_DIR/evtx
-    mkdir -p $EVTX_DIR
-
-    # import evtx and write them to import ingest pipeline
-    echo "- importing logs to Elasticsearch..."
-    evtx2es "${EVTX}" $HASH
-
-    # compare $START to $START_OLDEST
-    START=$(cat /nsm/import/evtx-start_oldest)
-    START_COMPARE=$(date -d $START +%s)
-    START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s)
-    if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then
-      START_OLDEST=$START
-    fi  
-
-    # compare $ENDNEXT to $END_NEWEST
-    END=$(cat /nsm/import/evtx-end_newest)
-    ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"`
-    ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s)
-    END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s)
-    if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then
-      END_NEWEST=$ENDNEXT
-    fi  
-
-    cp -f "${EVTX}" "${EVTX_DIR}"/data.evtx
-    chmod 644 "${EVTX_DIR}"/data.evtx
-
-  fi # end of valid evtx
-
-  echo
-
-done # end of for-loop processing evtx files
-
-# remove temp files
-echo "Cleaning up:"
-for TEMP_EVTX in ${TEMP_EVTXS[@]}; do
-  echo "- removing temporary evtx $TEMP_EVTX"
-  rm -f $TEMP_EVTX
-done
-
-# output final messages
-if [ "$INVALID_EVTXS" = "yes" ]; then
-  echo
-  echo "Please note!  One or more evtx was invalid!  You can scroll up to see which ones were invalid."
-fi
-
-START_OLDEST_FORMATTED=`date +%Y-%m-%d --date="$START_OLDEST"`
-START_OLDEST_SLASH=$(echo $START_OLDEST_FORMATTED | sed -e 's/-/%2F/g')
-END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
-
-if [ "$VALID_EVTXS" = "yes" ]; then
-cat << EOF
-
-Import complete!
-
-You can use the following hyperlink to view data in the time range of your import.  You can triple-click to quickly highlight the entire hyperlink and you can then copy it into your browser:
-https://{{ URLBASE }}/#/dashboards?q=import.id:${RUNID}%20%7C%20groupby%20-sankey%20event.dataset%20event.category%2a%20%7C%20groupby%20-pie%20event.category%20%7C%20groupby%20-bar%20event.module%20%7C%20groupby%20event.dataset%20%7C%20groupby%20event.module%20%7C%20groupby%20event.category%20%7C%20groupby%20observer.name%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC
-
-or you can manually set your Time Range to be (in UTC):
-From: $START_OLDEST_FORMATTED    To: $END_NEWEST
-
-Please note that it may take 30 seconds or more for events to appear in Security Onion Console.
-EOF
-fi

salt/common/tools/sbin/so-index-list

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-curl -K /opt/so/conf/elasticsearch/curl.config-X GET -k -L "https://localhost:9200/_cat/indices?v&s=index"

salt/common/tools/sbin/so-ip-update

@@ -49,10 +49,6 @@ if [ "$CONTINUE" == "y" ]; then
 		sed -i "s|$OLD_IP|$NEW_IP|g" $file
 	done
 
-	echo "Granting MySQL root user permissions on $NEW_IP"
-	docker exec -i so-mysql mysql --user=root --password=$(lookup_pillar_secret 'mysql') -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'$NEW_IP' IDENTIFIED BY '$(lookup_pillar_secret 'mysql')' WITH GRANT OPTION;" &> /dev/null
-	echo "Removing MySQL root user from $OLD_IP"
-	docker exec -i so-mysql mysql --user=root --password=$(lookup_pillar_secret 'mysql') -e "DROP USER 'root'@'$OLD_IP';" &> /dev/null
 	echo "Updating Kibana dashboards"
 	salt-call state.apply kibana.so_savedobjects_defaults -l info queue=True
 	

salt/common/tools/sbin/so-log-check

@@ -0,0 +1,261 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+RECENT_LOG_LINES=200
+EXCLUDE_STARTUP_ERRORS=N
+EXCLUDE_FALSE_POSITIVE_ERRORS=N
+EXCLUDE_KNOWN_ERRORS=N
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --exclude-connection-errors)
+            EXCLUDE_STARTUP_ERRORS=Y
+            ;;
+        --exclude-false-positives)
+            EXCLUDE_FALSE_POSITIVE_ERRORS=Y
+            ;;
+        --exclude-known-errors)
+            EXCLUDE_KNOWN_ERRORS=Y
+            ;;
+        --unknown)
+            EXCLUDE_STARTUP_ERRORS=Y
+            EXCLUDE_FALSE_POSITIVE_ERRORS=Y
+            EXCLUDE_KNOWN_ERRORS=Y
+            ;;
+        --recent-log-lines)
+            shift
+            RECENT_LOG_LINES=$1
+            ;;
+        *)
+            echo "Usage: $0 [options]"
+            echo ""
+            echo "where options are:"
+            echo "  --recent-log-lines N            looks at the most recent N log lines per file or container; defaults to 200"
+            echo "  --exclude-connection-errors     exclude errors caused by a recent server or container restart"
+            echo "  --exclude-false-positives       exclude logs that are known false positives"
+            echo "  --exclude-known-errors          exclude errors that are known and non-critical issues"
+            echo "  --unknown                       exclude everything mentioned above; only show unknown errors"
+            echo ""
+            echo "A non-zero return value indicates errors were found"
+            exit 1
+            ;;
+    esac
+    shift
+done
+
+echo "Security Onion Log Check - $(date)"
+echo "-------------------------------------------"
+echo ""
+echo "- RECENT_LOG_LINES:                $RECENT_LOG_LINES"
+echo "- EXCLUDE_STARTUP_ERRORS:          $EXCLUDE_STARTUP_ERRORS"
+echo "- EXCLUDE_FALSE_POSITIVE_ERRORS:   $EXCLUDE_FALSE_POSITIVE_ERRORS"
+echo "- EXCLUDE_KNOWN_ERRORS:            $EXCLUDE_KNOWN_ERRORS"
+echo ""
+
+function status() {
+    header "$1"
+}
+
+function exclude_container() {
+    name=$1
+
+    exclude_id=$(docker ps | grep "$name" | awk '{print $1}')
+    if [[ -n "$exclude_id" ]]; then
+        CONTAINER_IDS=$(echo $CONTAINER_IDS | sed -e "s/$exclude_id//g")
+        return $?
+    fi
+    return $?
+}
+
+function exclude_log() {
+    name=$1
+
+    cat /tmp/log_check_files | grep -v $name > /tmp/log_check_files.new
+    mv /tmp/log_check_files.new /tmp/log_check_files
+}
+
+function check_for_errors() {
+    if cat /tmp/log_check | grep -i error | grep -vEi "$EXCLUDED_ERRORS"; then
+        RESULT=1
+    fi
+}
+
+EXCLUDED_ERRORS="__LOG_CHECK_PLACEHOLDER_EXCLUSION__"
+
+if [[ $EXCLUDE_STARTUP_ERRORS == 'Y' ]]; then
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|database is locked"           # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|econnreset"                   # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unreachable"                  # server not yet ready (logstash waiting on elastic)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|shutdown process"             # server not yet ready (logstash waiting on elastic)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|contain valid certificates"   # server not yet ready (logstash waiting on elastic)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failedaction"                 # server not yet ready (logstash waiting on elastic)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no route to host"             # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|not running"                  # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unavailable"                  # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|request.py"                   # server not yet ready (python stack output)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|httperror"                    # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|servfail"                     # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connect"                      # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing shards"               # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to send metrics"       # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|broken pipe"                  # server not yet ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status: 502"                  # server not yet ready (nginx waiting on upstream)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeout exceeded"             # server not yet ready (telegraf waiting on elasticsearch)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|influxsize kbytes"            # server not yet ready (telegraf waiting on influx)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|expected field at"            # server not yet ready (telegraf waiting on health data)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|connection timed out"         # server not yet ready (telegraf plugin unable to connect)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|command timed out"            # server not yet ready (telegraf plugin waiting for script to finish)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cached the public key"        # server not yet ready (salt minion waiting on key acceptance)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|no ingest nodes"              # server not yet ready (logstash waiting on elastic)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to poll"               # server not yet ready (sensoroni waiting on soc)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|minions returned with non"    # server not yet ready (salt waiting on minions)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so_long_term"                 # server not yet ready (influxdb not yet setup)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|search_phase_execution_exception" # server not yet ready (elastalert running searches before ES is ready)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeout retrieving docker"    # Telegraf unable to reach Docker engine, rare
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeout retrieving container" # Telegraf unable to reach Docker engine, rare
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error while communicating"    # Elasticsearch MS -> HN "sensor" temporarily unavailable
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|tls handshake error"          # Docker registry container when new node comes onlines
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to get license information" # Logstash trying to contact ES before it's ready
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|process already finished"     # Telegraf script finished just as the auto kill timeout kicked in
+fi
+
+if [[ $EXCLUDE_FALSE_POSITIVE_ERRORS == 'Y' ]]; then
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|elastalert_status_error"      # false positive
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|elastalert_error"             # false positive
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error: '0'"                   # false positive
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|errors_index"                 # false positive
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noerror"                      # false positive
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|outofmemoryerror"             # false positive (elastic command line)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding component template"    # false positive (elastic security)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|adding index template"        # false positive (elastic security)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fs_errors"                    # false positive (suricata stats)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error-template"               # false positive (elastic templates)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|deprecated"                   # false positive (playbook)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|windows"                      # false positive (playbook)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|could cause errors"           # false positive (playbook)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_error.yml"                   # false positive (playbook)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|id.orig_h"                    # false positive (zeek test data)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|emerging-all.rules"           # false positive (error in rulename)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|invalid query input"          # false positive (Invalid user input in hunt query)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|example"                      # false positive (example test data)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|status 200"                   # false positive (request successful, contained error string in content)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|app_layer.error"              # false positive (suricata 7) in stats.log e.g. app_layer.error.imap.parser | Total | 0
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|is not an ip string literal"  # false positive (Open Canary logging out blank IP addresses)
+fi
+
+if [[ $EXCLUDE_KNOWN_ERRORS == 'Y' ]]; then
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|eof"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|raise"                        # redis/python generic stack line, rely on other lines for actual error
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fail\\(error\\)"              # redis/python generic stack line, rely on other lines for actual error
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|urlerror"                     # idstools connection timeout
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|timeouterror"                 # idstools connection timeout
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|_ml"                          # Elastic ML errors 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context canceled"             # elastic agent during shutdown
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|geoip databases update"       # airgap can't update GeoIP DB
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|filenotfounderror"            # bug in 2.4.10 filecheck salt state caused duplicate cronjobs
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|salt-minion-check"            # bug in early 2.4 place Jinja script in non-jinja salt dir causing cron output errors
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|monitoring.metrics"           # known issue with elastic agent casting the field incorrectly if an integer value shows up before a float
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|repodownload.conf"            # known issue with reposync on pre-2.4.20 
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|missing versions record"      # stenographer corrupt index
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|soc.field."                   # known ingest type collisions issue with earlier versions of SO
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error parsing signature"      # Malformed Suricata rule, from upstream provider
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sticky buffer has no matches" # Non-critical Suricata error
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unable to determine destination index stats"  # Elastic transform temporary error
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|cannot join on an empty table" # InfluxDB flux query, import nodes
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|exhausting result iterator"   # InfluxDB flux query mismatched table results (temporary data issue)
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to finish run"         # InfluxDB rare error, self-recoverable
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|iteration"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|communication packets"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|use of closed"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|bookkeeper"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|noindices"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|failed to start transient scope"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|so-user.lock exists"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|systemd-run"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|retcode: 1"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|telemetry-task"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|redisqueue"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|fleet_detail_query"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|num errors=0"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisioning/alerting"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisioning/notifiers"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|provisoning/plugins"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|active-responses.log"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|scanentropy"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integration policy"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|blob unknown"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|token required"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|zeekcaptureloss"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|unable to create detection"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|error installing new prebuilt rules"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parent.error"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|req.LocalMeta.host.ip"        # known issue in GH
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|sendmail"                     # zeek
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|stats.log"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Unknown column"               # Elastalert errors from running EQL queries
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|parsing_exception"            # Elastalert EQL parsing issue. Temp.
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|context deadline exceeded"
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|Error running query:"         # Specific issues with detection rules
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|detect-parse"                 # Suricata encountering a malformed rule
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|integrity check failed"       # Detections: Exclude false positive due to automated testing
+    EXCLUDED_ERRORS="$EXCLUDED_ERRORS|syncErrors"                   # Detections: Not an actual error
+fi
+
+RESULT=0
+
+# Check Security Onion container stdout/stderr logs
+CONTAINER_IDS=$(docker ps -q)
+exclude_container so-kibana   # kibana error logs are too verbose with large varieties of errors most of which are temporary
+exclude_container so-idstools # ignore due to known issues and noisy logging
+exclude_container so-playbook # Playbook is removed as of 2.4.70, disregard output in stopped containers
+exclude_container so-mysql    # MySQL is removed as of 2.4.70, disregard output in stopped containers
+exclude_container so-soctopus # Soctopus is removed as of 2.4.70, disregard output in stopped containers
+
+for container_id in $CONTAINER_IDS; do
+    container_name=$(docker ps --format json | jq ". | select(.ID==\"$container_id\")|.Names")
+    status "Checking container $container_name"
+    docker logs -n $RECENT_LOG_LINES $container_id > /tmp/log_check 2>&1
+    check_for_errors
+done
+
+# Check Security Onion related log files
+find /opt/so/log/ /nsm -name \*.log > /tmp/log_check_files
+if [[ -f /var/log/cron ]]; then
+    echo "/var/log/cron" >> /tmp/log_check_files
+fi
+exclude_log "kibana.log"   # kibana error logs are too verbose with large varieties of errors most of which are temporary
+exclude_log "spool"        # disregard zeek analyze logs as this is data specific
+exclude_log "import"       # disregard imported test data the contains error strings
+exclude_log "update.log"   # ignore playbook updates due to several known issues
+exclude_log "cron-cluster-delete.log" # ignore since Curator has been removed
+exclude_log "cron-close.log" # ignore since Curator has been removed
+exclude_log "curator.log"  # ignore since Curator has been removed
+exclude_log "playbook.log" # Playbook is removed as of 2.4.70, logs may still be on disk
+exclude_log "mysqld.log"   # MySQL is removed as of 2.4.70, logs may still be on disk
+exclude_log "soctopus.log" # Soctopus is removed as of 2.4.70, logs may still be on disk
+exclude_log "agentstatus.log" # ignore this log since it tracks agents in error state
+exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable
+
+for log_file in $(cat /tmp/log_check_files); do
+    status "Checking log file $log_file"
+    tail -n $RECENT_LOG_LINES $log_file > /tmp/log_check
+    check_for_errors
+done
+
+# Cleanup temp files
+rm -f /tmp/log_check_files
+rm -f /tmp/log_check
+
+if [[ $RESULT -eq 0 ]]; then
+    echo -e "\nResult: No errors found"
+else
+    echo -e "\nResult: One or more errors found"
+fi
+
+exit $RESULT

salt/common/tools/sbin/so-logstash-events

@@ -1,17 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% set MAININT = salt['pillar.get']('host:mainint') -%}
-{% set NODEIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] -%}
-
-. /usr/sbin/so-common
-
-if [ "$1" == "" ]; then
-        for i in $(curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines | jq '. | to_entries | .[].key' | sed 's/\"//g'); do echo ${i^}:; curl -s localhost:9600/_node/stats | jq .pipelines.$i.events; done
-else
-        curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1.events
-fi

salt/common/tools/sbin/so-logstash-get-unparsed

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-docker exec -it so-redis redis-cli llen logstash:unparsed

salt/common/tools/sbin/so-luks-tpm-regen

@@ -0,0 +1,98 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0."
+
+set -e
+# This script is intended to be used in the case the ISO install did not properly setup TPM decrypt for LUKS partitions at boot.
+if [ -z $NOROOT ]; then
+	# Check for prerequisites
+	if [ "$(id -u)" -ne 0 ]; then
+		echo "This script must be run using sudo!"
+		exit 1
+	fi
+fi
+ENROLL_TPM=N
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --enroll-tpm)
+      ENROLL_TPM=Y
+      ;;
+    *)
+      echo "Usage: $0 [options]"
+      echo ""
+      echo "where options are:"
+      echo "  --enroll-tpm            for when TPM enrollment was not selected during ISO install."
+      echo ""
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+check_for_tpm() {
+  echo -n "Checking for TPM: "
+  if [ -d /sys/class/tpm/tpm0 ]; then
+    echo -e "tpm0 found."
+    TPM="yes"
+    # Check if TPM is using sha1 or sha256
+    if [ -d /sys/class/tpm/tpm0/pcr-sha1 ]; then
+      echo -e "TPM is using sha1.\n"
+      TPM_PCR="sha1"
+    elif [ -d /sys/class/tpm/tpm0/pcr-sha256 ]; then
+      echo -e "TPM is using sha256.\n"
+      TPM_PCR="sha256"
+    fi
+  else
+    echo -e "No TPM found.\n"
+    exit 1
+  fi
+}
+
+check_for_luks_partitions() {
+  echo "Checking for LUKS partitions"
+  for part in $(lsblk -o NAME,FSTYPE -ln | grep crypto_LUKS | awk '{print $1}'); do
+    echo "Found LUKS partition: $part"
+    LUKS_PARTITIONS+=("$part")
+  done
+  if [ ${#LUKS_PARTITIONS[@]} -eq 0 ]; then
+    echo -e "No LUKS partitions found.\n"
+    exit 1
+  fi
+  echo ""
+}
+
+enroll_tpm_in_luks() {
+  read -s -p "Enter the LUKS passphrase used during ISO install: " LUKS_PASSPHRASE
+  echo ""
+  for part in "${LUKS_PARTITIONS[@]}"; do
+    echo "Enrolling TPM for LUKS device: /dev/$part"
+    if [ "$TPM_PCR" == "sha1" ]; then
+      clevis luks bind -d /dev/$part tpm2 '{"pcr_bank":"sha1","pcr_ids":"7"}' <<< $LUKS_PASSPHRASE
+    elif [ "$TPM_PCR" == "sha256" ]; then
+      clevis luks bind -d /dev/$part tpm2 '{"pcr_bank":"sha256","pcr_ids":"7"}' <<< $LUKS_PASSPHRASE
+    fi
+  done
+ }
+
+regenerate_tpm_enrollment_token() {
+  for part in "${LUKS_PARTITIONS[@]}"; do
+    clevis luks regen -d /dev/$part -s 1 -q
+  done
+}
+
+check_for_tpm
+check_for_luks_partitions
+
+if [[ $ENROLL_TPM == "Y" ]]; then
+  enroll_tpm_in_luks
+else
+  regenerate_tpm_enrollment_token
+fi
+
+echo "Running dracut"
+dracut -fv
+echo -e "\nTPM configuration complete. Reboot the system to verify the TPM is correctly decrypting the LUKS partition(s) at boot.\n"

salt/common/tools/sbin/so-minion

@@ -1,365 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-if [ -f /usr/sbin/so-common ]; then
-	. /usr/sbin/so-common
-fi
-
-if [ "$(id -u)" -ne 0 ]; then
-	echo "This script must be run using sudo!"
-	exit 1
-fi
-
-if [[ $# -lt 1 ]]; then
-  echo "Usage: $0 -o=<operation> -m=[id]"
-  echo ""
-  echo " where <operation> is one of the following:"
-  echo ""
-  echo "     list: Lists all keys with hashes"
-  echo "   accept: Accepts a new key and adds the minion files"
-  echo "   delete: Removes the key and deletes the minion files"
-  echo "   reject: Rejects a key"
-  echo "     test: Perform minion test"
-  echo ""
-  exit 1
-fi
-
-for i in "$@"; do
-	case $i in
-		-o=*|--operation=*)
-			OPERATION="${i#*=}"
-			shift 
-			;;
-		-m=*|--minionid=*)
-			MINION_ID="${i#*=}"
-			shift 
-			;;
-		-e=*|--esheap=*)
-			ES_HEAP_SIZE="${i#*=}"
-			shift
-			;;
-		-n=*|--mgmtnic=*)
-			MNIC="${i#*=}"
-			shift
-			;;
-		-d=*|--description=*)
-			NODE_DESCRIPTION="${i#*=}"
-			shift
-			;;
-		-a=*|--monitor=*)
-			INTERFACE="${i#*=}"
-			shift
-			;;
-		-i=*|--ip=*)
-			MAINIP="${i#*=}"
-			shift
-			;;		
-		-*|--*)
-			echo "Unknown option $i"
-			exit 1
-			;;
-		*)
-		;;
-	esac
-done
-
-PILLARFILE=/opt/so/saltstack/local/pillar/minions/$MINION_ID.sls
-ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls
-
-function getinstallinfo() {
-	# Pull from file
-	INSTALLVARS=$(sudo salt "$MINION_ID" cp.get_file_str /opt/so/install.txt --out=newline_values_only)
-	source <(echo $INSTALLVARS)
-}
-
-function testminion() {
-	# Always run on the host, since this is going to be the manager of a distributed grid, or an eval/standalone.
-	# Distributed managers must run this in order for the sensor nodes to have access to the so-tcpreplay image.
-	so-test
-	result=$?
-
-	# If this so-minion script is not running on the given minion ID, run so-test remotely on the sensor as well
-	local_id=$(lookup_grain id)
-	if [[ ! "$local_id" =~ "${MINION_ID}_" ]]; then
-		salt "$MINION_ID" cmd.run 'so-test'
-		result=$?
-	fi
-
-	exit $result
-}
-
-function listminions() {
-	salt-key list -F --out=json
-	exit $?
-}
-
-function rejectminion() {
-	salt-key -y -r $MINION_ID
-	exit $?
-}
-
-function acceptminion() {
-	salt-key -y -a $MINION_ID
-}
-
-function deleteminion() {
-	salt-key -y -d $MINION_ID
-}
-
-function deleteminionfiles () {
-	rm -f $PILLARFILE
-	rm -f $ADVPILLARFILE
-}
-
-# Create the minion file
-function create_minion_files() {
-	mkdir -p /opt/so/saltstack/local/pillar/minions
-    touch $ADVPILLARFILE
-	if [ -f "$PILLARFILE" ]; then
-		rm $PILLARFILE
-	fi
-}
-
-# Add Elastic settings to the minion file
-function add_elastic_to_minion() {
-    printf '%s\n'\
-		"elasticsearch:"\
-		"  esheap: '$ES_HEAP_SIZE'"\
-		"  " >> $PILLARFILE
-}
-
-# Add Elastic Fleet Server settings to the minion file
-function add_fleet_to_minion() {
-
-    # Create ES Token for Fleet server (Curl to Kibana API)
-	# TODO: Add error handling
-    ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' |  jq -r .value)
-
-    # Write out settings to minion file
-    printf '%s\n'\
-		"elasticfleet:"\
-		"  server:"\
-		"    es_token: '$ESTOKEN'"\
-		"  " >> $PILLARFILE
-}
-
-
-# Add IDH Services info to the minion file
-function add_idh_to_minion() {
-	printf '%s\n'\
-	"idh:"\
-	"  restrict_management_ip: $IDH_MGTRESTRICT"\
-	"  services:" >> "$PILLARFILE"
-	IFS=',' read -ra IDH_SERVICES_ARRAY <<< "$IDH_SERVICES"
-	for service in ${IDH_SERVICES_ARRAY[@]}; do
-	  echo "    - $service" | tr '[:upper:]' '[:lower:]' | tr -d '"' >> "$PILLARFILE"
-	done
-}
-
-function add_logstash_to_minion() {
-	# Create the logstash advanced pillar
-	printf '%s\n'\
-		"logstash_settings:"\
-		"  ls_host: '$LSHOSTNAME'"\
-		"  ls_pipeline_batch_size: 125"\
-		"  ls_input_threads: 1"\
-		"  lsheap: $LSHEAP"\
-		"  ls_pipeline_workers: $CPUCORES"\
-		"  " >> $PILLARFILE
-}
-
-# Analyst Workstation
-function add_analyst_to_minion() {
-    printf '%s\n'\
-		"host:"\
-		"  mainint: '$MNIC'"\
-		"workstation:"\
-		"  gui:"\
-		"    enabled: true"\
-		"sensoroni:"\
-		"  node_description: '${NODE_DESCRIPTION//\'/''}'" >> $PILLARFILE
-}
-
-# Add basic host info to the minion file
-function add_host_to_minion() {
-    printf '%s\n'\
-    "host:"\
-	"  mainip: '$MAINIP'"\
-    "  mainint: '$MNIC'" >> $PILLARFILE
-}
-
-# Add sensoroni specific information - Can we pull node_adrees from the host pillar?
-function add_sensoroni_to_minion() {
-
-	printf '%s\n'\
-    "sensoroni:"\
-		"  node_description: '${NODE_DESCRIPTION//\'/''}'"\
-        "  " >> $PILLARFILE
-}	
-
-# Sensor settings for the minion pillar
-function add_sensor_to_minion() {
-	echo "sensor:" >> $PILLARFILE
-	echo "  interface: '$INTERFACE'" >> $PILLARFILE
-	echo "  mtu: 9000" >> $PILLARFILE
-	echo "zeek:" >> $PILLARFILE
-	echo "    config:" >> $PILLARFILE
-	echo "        node:" >> $PILLARFILE
-	echo "            lb_procs: '$CORECOUNT'" >> $PILLARFILE
-	echo "suricata:" >> $PILLARFILE
-	echo "    config:" >> $PILLARFILE
-	echo "        af-packet:" >> $PILLARFILE
-	echo "            threads: '$CORECOUNT'" >> $PILLARFILE
-	echo "pcap:" >> $PILLARFILE
-	echo "    enabled: True" >> $PILLARFILE
-}
-
-function create_fleet_policy() {
-
-	JSON_STRING=$( jq -n \
-					--arg NAME "FleetServer_$LSHOSTNAME" \
-					--arg DESC "Fleet Server - $LSHOSTNAME" \
-					'{"name": $NAME,"id":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":1209600,"has_fleet_server":true}'
-					)
-
-	# Create Fleet Sever Policy
-	curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/agent_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-
-	JSON_STRING_UPDATE=$( jq -n \
-					--arg NAME "FleetServer_$LSHOSTNAME" \
-					--arg DESC "Fleet Server - $LSHOSTNAME" \
-					'{"name":$NAME,"description":$DESC,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":1209600,"data_output_id":"so-manager_elasticsearch"}'
-					)
-
-	# Update Fleet Policy - ES Output
-	curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_$LSHOSTNAME" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING_UPDATE"
-}
-
-function update_fleet_host_urls() {
-	# Query for current Fleet Host URLs & append New Fleet Node Hostname & IP
-    JSON_STRING=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/fleet_server_hosts' | jq --arg HOSTNAME "https://$LSHOSTNAME:8220" --arg IP "https://$MAINIP:8220" -r '.items[].host_urls += [ $HOSTNAME, $IP ] | {"name":"Default","host_urls": .items[].host_urls,"is_default":true,"proxy_id":null}')
-
-	# Update Fleet Host URLs
-	curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/fleet_server_hosts/fleet-default-fleet-server-host" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-}
-
-function update_logstash_outputs() {
-	# Query for current Logstash outputs & append New Fleet Node Hostname & IP
-    JSON_STRING=$(curl -K /opt/so/conf/elasticsearch/curl.config 'http://localhost:5601/api/fleet/outputs/so-manager_logstash' | jq --arg HOSTNAME "$LSHOSTNAME:5055" --arg IP "$MAINIP:5055" -r '.item.hosts += [ $HOSTNAME, $IP ] | {"name":"grid-logstash","type":"logstash","hosts": .item.hosts,"is_default":true,"is_default_monitoring":true,"config_yaml":""}')
-
-	# Update Logstash Outputs
-    curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/outputs/so-manager_logstash" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-}
-
-function updateMine() {
-	salt "$MINION_ID" mine.send network.ip_addrs interface="$MNIC"
-}
-function apply_ES_state() {
-	salt-call state.apply elasticsearch concurrent=True
-}
-function createEVAL() {
-	add_elastic_to_minion
-	add_logstash_to_minion
-	add_sensor_to_minion
-}
-
-function createFLEET() {
-	add_fleet_to_minion
-	add_logstash_to_minion
-	create_fleet_policy
-	update_fleet_host_urls
-	update_logstash_outputs
-}
-
-function createIDH() {
-	add_idh_to_minion
-}
-
-function createIMPORT() {
-	add_elastic_to_minion
-	add_logstash_to_minion
-	add_sensor_to_minion
-}
-
-function createHEAVYNODE() {
-	add_elastic_to_minion
-	add_logstash_to_minion
-	add_sensor_to_minion
-}
-
-function createMANAGER() {
-	add_elastic_to_minion
-	add_logstash_to_minion
-}
-
-function createMANAGERSEARCH() {
-	add_elastic_to_minion
-	add_logstash_to_minion
-}
-
-function createSENSOR() {
-	add_sensor_to_minion
-}
-
-function createSEARCHNODE() {
-	add_elastic_to_minion
-	add_logstash_to_minion
-	updateMine
-	apply_ES_state
-}
-
-function createSTANDALONE() {
-	add_elastic_to_minion
-	add_logstash_to_minion
-	add_sensor_to_minion
-}
-
-function testConnection() {
-	retry 15 3 "salt '$MINION_ID' test.ping" True
-	local ret=$?
-	if [[ $ret != 0 ]]; then
-		echo "The Minion has been accepted but is not online. Try again later"
-		echo "Deleting the key"
-		deleteminion
-		exit 1
-	fi
-}
-
-if [[ "$OPERATION" = 'list' ]]; then
-	listminions
-fi
-
-if [[ "$OPERATION" = 'delete' ]]; then
-	deleteminionfiles
-	deleteminion
-fi
-
-if [[ "$OPERATION" = 'add' || "$OPERATION" = 'setup' ]]; then
-	# Skip this if its setup
-	if [ $OPERATION != 'setup' ]; then
-		# Accept the salt key
-		acceptminion
-		# Test to see if the minion was accepted
-		testConnection
-		# Pull the info from the file to build what is needed
-		getinstallinfo
-	fi
-	# Check to see if nodetype is set
-	if [ -z $NODETYPE ]; then
-		echo "No node type specified"
-		exit 1
-	fi
-	create_minion_files
-	add_host_to_minion
-	add_sensoroni_to_minion
-	create$NODETYPE
-	echo "Minion file created for $MINION_ID"	
-fi
-
-if [[ "$OPERATION" = 'test' ]]; then
-	testminion
-fi

salt/common/tools/sbin/so-mysql-start

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-start mysql $1

salt/common/tools/sbin/so-mysql-stop

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-stop mysql $1

salt/common/tools/sbin/so-nodered-restart

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-restart nodered $1

salt/common/tools/sbin/so-nodered-stop

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-stop nodered $1

salt/common/tools/sbin/so-nsm-clear

@@ -41,8 +41,13 @@ done
 if [ $SKIP -ne 1 ]; then
         # Inform user we are about to delete all data
         echo
-        echo "This script will delete all NIDS data (PCAP, Suricata, Zeek)" 
-        echo "If you would like to proceed, please type "AGREE" and hit ENTER."
+        echo "This script will delete all NSM data from /nsm."
+        echo
+        echo "This includes Suricata data, Zeek data, and full packet capture (PCAP)."
+        echo
+        echo "This will NOT delete any Suricata or Zeek logs that have already been ingested into Elasticsearch."
+        echo
+        echo "If you would like to proceed, then type AGREE and press ENTER."
         echo
         # Read user input
         read INPUT
@@ -54,8 +59,8 @@ delete_pcap() {
   [ -d $PCAP_DATA ] && so-pcap-stop && rm -rf $PCAP_DATA/* && so-pcap-start
 }
 delete_suricata() {
-  SURI_LOG="/opt/so/log/suricata/eve.json"
-  [ -f $SURI_LOG ] && so-suricata-stop && rm -f $SURI_LOG && so-suricata-start
+  SURI_LOG="/nsm/suricata/"
+  [ -d $SURI_LOG ] && so-suricata-stop && rm -rf $SURI_LOG/* && so-suricata-start
 }
 delete_zeek() {
   ZEEK_LOG="/nsm/zeek/logs/"

salt/common/tools/sbin/so-playbook-reset

@@ -1,22 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-salt-call state.apply playbook.db_init,playbook
-
-/usr/sbin/so-soctopus-restart
-
-salt-call state.apply playbook,playbook.automation_user_create
-
-/usr/sbin/so-soctopus-restart
-
-echo "Importing Plays - NOTE: this will continue after installation finishes and could take an hour or more. Rebooting while the import is in progress will delay playbook imports."
-sleep 5
-so-playbook-ruleupdate >> /root/setup_playbook_rule_update.log 2>&1 &

salt/common/tools/sbin/so-playbook-restart

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-restart playbook $1

salt/common/tools/sbin/so-playbook-ruleupdate

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-docker exec so-soctopus python3 playbook_bulk-update.py

salt/common/tools/sbin/so-playbook-sigma-refresh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-if ! [ -f /opt/so/state/playbook_regen_plays ] || [ "$1" = "--force" ]; then
-
-    echo "Refreshing Sigma & regenerating plays... "
-
-    # Regenerate ElastAlert & update Plays
-    docker exec so-soctopus python3 playbook_play-update.py
-
-    # Delete current Elastalert Rules
-    rm /opt/so/rules/elastalert/playbook/*.yaml
-
-    # Regenerate Elastalert Rules
-    so-playbook-sync
-    
-    # Create state file
-    touch /opt/so/state/playbook_regen_plays
-else
-  printf "\nState file found, exiting...\nRerun with --force to override.\n"
-fi

salt/common/tools/sbin/so-playbook-start

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-start playbook $1

salt/common/tools/sbin/so-playbook-stop

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-stop playbook $1

salt/common/tools/sbin/so-playbook-sync

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-# Check to see if we are already running
-NUM_RUNNING=$(pgrep -cf "/bin/bash /usr/sbin/so-playbook-sync")
-[ "$NUM_RUNNING" -gt 1 ] && echo "$(date) - $NUM_RUNNING Playbook sync processes running...exiting." && exit 0
-
-docker exec so-soctopus python3 playbook_play-sync.py

salt/common/tools/sbin/so-raid-status

@@ -1,109 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-appliance_check() {
-  {%- if salt['grains.get']('sosmodel', '') %}
-  APPLIANCE=1
-  {%- if grains['sosmodel'] in ['SO2AMI01', 'SO2GCI01', 'SO2AZI01'] %}
-  exit 0
-  {%- endif %}
-  DUDEYOUGOTADELL=$(dmidecode |grep Dell)
-  if [[ -n $DUDEYOUGOTADELL ]]; then
-  APPTYPE=dell
-  else
-  APPTYPE=sm
-  fi
-  mkdir -p /opt/so/log/raid
-
-  {%- else %}
-  echo "This is not an appliance"
-  exit 0
-  {%- endif %}
-}
-
-check_nsm_raid() {
-  PERCCLI=$(/opt/raidtools/perccli/perccli64 /c0/v0 show|grep RAID|grep Optl)
-  MEGACTL=$(/opt/raidtools/megasasctl |grep optimal)
-
-  if [[ $APPLIANCE == '1' ]]; then
-    if [[ -n $PERCCLI ]]; then
-      HWRAID=0
-    elif [[ -n $MEGACTL ]]; then
-      HWRAID=0
-    else
-      HWRAID=1
-    fi
-
-  fi
-
-}
-
-check_boss_raid() {
-  MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional)
-
-  if [[ -n $DUDEYOUGOTADELL ]]; then
-    if [[ -n $MVCLI ]]; then
-      BOSSRAID=0
-    else
-      BOSSRAID=1
-    fi
-  fi
-}
-
-check_software_raid() {
-  if [[ -n $DUDEYOUGOTADELL ]]; then
-    SWRC=$(grep "_" /proc/mdstat)
-
-    if [[ -n $SWRC ]]; then
-        # RAID is failed in some way
-        SWRAID=1
-    else
-        SWRAID=0
-    fi
-  fi
-}
-
-# This script checks raid status if you use SO appliances
-
-# See if this is an appliance
-
-appliance_check
-check_nsm_raid
-check_boss_raid
-{%- if salt['grains.get']('sosmodel', '') %}
-{%- if grains['sosmodel'] in ['SOSMN', 'SOSSNNV'] %}
-check_software_raid
-{%- endif %}
-{%- endif %}
-
-if [[ -n $SWRAID ]]; then
-  if [[ $SWRAID == '0' && $BOSSRAID == '0' ]]; then
-    RAIDSTATUS=0
-  else
-    RAIDSTATUS=1
-  fi
-elif [[ -n $DUDEYOUGOTADELL ]]; then
-  if [[ $BOSSRAID == '0' && $HWRAID == '0' ]]; then
-    RAIDSTATUS=0
-  else
-    RAIDSTATUS=1
-  fi
-elif [[ "$APPTYPE" == 'sm' ]]; then
-  if [[ -n "$HWRAID" ]]; then
-    RAIDSTATUS=0
-  else
-    RAIDSTATUS=1
-  fi
-fi
-
-echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log
-
-

salt/common/tools/sbin/so-restart

@@ -24,6 +24,7 @@ if [ $# -ge 1 ]; then
 
         case $1 in
                 "steno") docker stop so-steno && docker rm so-steno && salt-call state.apply pcap queue=True;;
+                "elastic-fleet") docker stop so-elastic-fleet && docker rm so-elastic-fleet && salt-call state.apply elasticfleet queue=True;;
                 *)  docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
         esac
 else

salt/common/tools/sbin/so-rule

@@ -1,454 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-""" 
-Local exit codes:
- - General error: 1
- - Invalid argument: 2
- - File error: 3
-"""
-
-import sys, os, subprocess, argparse, signal
-import copy
-import re
-import textwrap
-import yaml
-
-minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions'
-salt_proc: subprocess.CompletedProcess = None
-
-
-def print_err(string: str):
-  print(string, file=sys.stderr)
-
-
-def check_apply(args: dict, prompt: bool = True):
-  if args.apply:
-    print('Configuration updated. Applying changes:')
-    return apply()
-  else:
-    if prompt:
-      message = 'Configuration updated. Would you like to apply your changes now? (y/N) '
-      answer = input(message)
-      while answer.lower() not in [ 'y', 'n', '' ]:
-        answer = input(message)
-      if answer.lower() in [ 'n', '' ]:
-        return 0
-      else:
-        print('Applying changes:')
-        return apply()
-    else:
-      return 0
-
-
-def apply():
-  salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', 'idstools.sync_files', 'queue=True']
-  update_cmd = ['so-rule-update']
-  print('Syncing config files...')
-  cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL)
-  if cmd.returncode == 0:
-    print('Updating rules...')
-    return subprocess.run(update_cmd).returncode
-  else:
-    return cmd.returncode
-
-
-def find_minion_pillar() -> str:
-  regex = '^.*_(manager|managersearch|standalone|import|eval)\.sls$'
-  
-  result = []
-  for root, _, files in os.walk(minion_pillar_dir):
-    for f_minion_id in files:
-      if re.search(regex, f_minion_id):
-        result.append(os.path.join(root, f_minion_id))
-  
-  if len(result) == 0:
-    print_err('Could not find manager-type pillar (eval, standalone, manager, managersearch, import). Are you running this script on the manager?')
-    sys.exit(3)
-  elif len(result) > 1:
-    res_arr = []
-    for r in result:
-      res_arr.append(f'\"{r}\"')
-    res_str = ', '.join(res_arr)
-    print_err('(This should not happen, the system is in an error state if you see this message.)\n')
-    print_err('More than one manager-type pillar exists, minion id\'s listed below:')
-    print_err(f'  {res_str}')
-    sys.exit(3)
-  else:
-    return result[0]
-
-
-def read_pillar(pillar: str):
-  try:
-    with open(pillar, 'r') as f:
-      loaded_yaml = yaml.safe_load(f.read())
-      if loaded_yaml is None:
-        print_err(f'Could not parse {pillar}')
-        sys.exit(3)
-      return loaded_yaml
-  except:
-    print_err(f'Could not open {pillar}')
-    sys.exit(3)
-
-
-def write_pillar(pillar: str, content: dict):
-  try:
-    sids = content['idstools']['sids']
-    if sids['disabled'] is not None:
-      if len(sids['disabled']) == 0: sids['disabled'] = None
-    if sids['enabled'] is not None:
-      if len(sids['enabled']) == 0: sids['enabled'] = None
-    if sids['modify'] is not None:
-      if len(sids['modify']) == 0: sids['modify'] = None
-
-    with open(pillar, 'w') as f:
-      return yaml.dump(content, f, default_flow_style=False)
-  except Exception as e:
-    print_err(f'Could not open {pillar}')
-    sys.exit(3)
-
-
-def check_sid_pattern(sid_pattern: str):
-  message = f'SID {sid_pattern} is not valid, did you forget the \"re:\" prefix for a regex pattern?'
-
-  if sid_pattern.startswith('re:'):
-    r_string = sid_pattern[3:]
-    if not valid_regex(r_string):
-      print_err('Invalid regex pattern.')
-      return False
-    else:
-      return True
-  else:
-    sid: int
-    try:
-      sid = int(sid_pattern)
-    except:
-      print_err(message)
-      return False
-
-    if sid >= 0:
-      return True
-    else:
-      print_err(message)
-      return False
-
-
-def valid_regex(pattern: str):
-  try:
-    re.compile(pattern)
-    return True
-  except re.error:
-    return False
-
-
-def sids_key_exists(pillar: dict, key: str):
-  return key in pillar.get('idstools', {}).get('sids', {})
-
-
-def rem_from_sids(pillar: dict, key: str, val: str, optional = False):
-  pillar_dict = copy.deepcopy(pillar)
-  arr = pillar_dict['idstools']['sids'][key]
-  if arr is None or val not in arr:
-    if not optional: print(f'{val} already does not exist in {key}')
-  else:
-    pillar_dict['idstools']['sids'][key].remove(val)
-  return pillar_dict
-
-
-def add_to_sids(pillar: dict, key: str, val: str, optional = False):
-  pillar_dict = copy.deepcopy(pillar)
-  if pillar_dict['idstools']['sids'][key] is None: 
-    pillar_dict['idstools']['sids'][key] = []
-  if val in pillar_dict['idstools']['sids'][key]:
-    if not optional: print(f'{val} already exists in {key}')
-  else:
-    pillar_dict['idstools']['sids'][key].append(val)
-  return pillar_dict
-
-
-def add_rem_disabled(args: dict):
-  global salt_proc
-
-  if not check_sid_pattern(args.sid_pattern):
-    return 2
-
-  pillar_dict = read_pillar(args.pillar)
-
-  if not sids_key_exists(pillar_dict, 'disabled'):
-    pillar_dict['idstools']['sids']['disabled'] = None
-  
-  if args.remove:
-    temp_pillar_dict = rem_from_sids(pillar_dict, 'disabled', args.sid_pattern)
-  else:
-    temp_pillar_dict = add_to_sids(pillar_dict, 'disabled', args.sid_pattern)
-
-  if temp_pillar_dict['idstools']['sids']['disabled'] == pillar_dict['idstools']['sids']['disabled']:
-    salt_proc = check_apply(args, prompt=False)
-    return salt_proc
-  else:
-    pillar_dict = temp_pillar_dict
-
-  if not args.remove:
-    if sids_key_exists(pillar_dict, 'enabled'):
-      pillar_dict = rem_from_sids(pillar_dict, 'enabled', args.sid_pattern, optional=True)
-
-    modify = pillar_dict.get('idstools', {}).get('sids', {}).get('modify')
-    if modify is not None:
-      rem_candidates = []
-      for action in modify:
-        if action.startswith(f'{args.sid_pattern} '):
-          rem_candidates.append(action)
-      if len(rem_candidates) > 0:
-        for item in rem_candidates:
-          print(f' - {item}')
-        answer = input(f'The above modify actions contain {args.sid_pattern}. Would you like to remove them? (Y/n) ')
-        while answer.lower() not in [ 'y', 'n', '' ]:
-          for item in rem_candidates:
-            print(f' - {item}')
-          answer = input(f'The above modify actions contain {args.sid_pattern}. Would you like to remove them? (Y/n) ')
-        if answer.lower() in [ 'y', '' ]:
-          for item in rem_candidates:
-            modify.remove(item)
-          pillar_dict['idstools']['sids']['modify'] = modify
-
-  write_pillar(pillar=args.pillar, content=pillar_dict)
-  
-  salt_proc = check_apply(args)
-  return salt_proc
-
-
-def list_disabled_rules(args: dict):
-  pillar_dict = read_pillar(args.pillar)
-
-  disabled = pillar_dict.get('idstools', {}).get('sids', {}).get('disabled')
-  if disabled is None:
-    print('No rules disabled.')
-    return 0
-  else:
-    print('Disabled rules:')
-    for rule in disabled:
-      print(f'  - {rule}')
-    return 0
-
-
-def add_rem_enabled(args: dict):
-  global salt_proc
-
-  if not check_sid_pattern(args.sid_pattern):
-    return 2
-
-  pillar_dict = read_pillar(args.pillar)
-
-  if not sids_key_exists(pillar_dict, 'enabled'):
-    pillar_dict['idstools']['sids']['enabled'] = None
-
-  if args.remove:
-    temp_pillar_dict = rem_from_sids(pillar_dict, 'enabled', args.sid_pattern)
-  else:
-    temp_pillar_dict = add_to_sids(pillar_dict, 'enabled', args.sid_pattern)
-  
-  if temp_pillar_dict['idstools']['sids']['enabled'] == pillar_dict['idstools']['sids']['enabled']:
-    salt_proc = check_apply(args, prompt=False)
-    return salt_proc
-  else:
-    pillar_dict = temp_pillar_dict
-
-  if not args.remove:
-    if sids_key_exists(pillar_dict, 'disabled'):
-      pillar_dict = rem_from_sids(pillar_dict, 'disabled', args.sid_pattern, optional=True)
-  
-  write_pillar(pillar=args.pillar, content=pillar_dict)
-
-  salt_proc = check_apply(args)
-  return salt_proc
-
-
-def list_enabled_rules(args: dict):
-  pillar_dict = read_pillar(args.pillar)
-
-  enabled = pillar_dict.get('idstools', {}).get('sids', {}).get('enabled')
-  if enabled is None:
-    print('No rules explicitly enabled.')
-    return 0
-  else:
-    print('Enabled rules:')
-    for rule in enabled:
-      print(f'  - {rule}')
-    return 0
-
-
-def add_rem_modify(args: dict):
-  global salt_proc
-
-  if not check_sid_pattern(args.sid_pattern):
-    return 2
-
-  if not valid_regex(args.search_term):
-    print_err('Search term is not a valid regex pattern.')
-
-  string_val = f'{args.sid_pattern} \"{args.search_term}\" \"{args.replace_term}\"'
-
-  pillar_dict = read_pillar(args.pillar)
-
-  if not sids_key_exists(pillar_dict, 'modify'):
-    pillar_dict['idstools']['sids']['modify'] = None
-
-  if args.remove:
-    temp_pillar_dict = rem_from_sids(pillar_dict, 'modify', string_val)
-  else:
-    temp_pillar_dict = add_to_sids(pillar_dict, 'modify', string_val)
-    
-  if temp_pillar_dict['idstools']['sids']['modify'] == pillar_dict['idstools']['sids']['modify']:
-    salt_proc = check_apply(args, prompt=False)
-    return salt_proc
-  else:
-    pillar_dict = temp_pillar_dict
-
-  # TODO: Determine if a rule should be removed from disabled if modified.
-  if not args.remove:
-    if sids_key_exists(pillar_dict, 'disabled'):
-      pillar_dict = rem_from_sids(pillar_dict, 'disabled', args.sid_pattern, optional=True) 
-
-  write_pillar(pillar=args.pillar, content=pillar_dict)
-  
-  salt_proc = check_apply(args)
-  return salt_proc
-  
-
-def list_modified_rules(args: dict):
-  pillar_dict = read_pillar(args.pillar)
-
-  modify = pillar_dict.get('idstools', {}).get('sids', {}).get('modify')
-  if modify is None:
-    print('No rules currently modified.')
-    return 0
-  else:
-    print('Modified rules + modifications:')
-    for rule in modify:
-      print(f'  - {rule}')
-    return 0
-
-
-def sigint_handler(*_):
-  print('Exiting gracefully on Ctrl-C')
-  if salt_proc is not None: salt_proc.send_signal(signal.SIGINT)
-  sys.exit(0)
-
-
-def main():
-  signal.signal(signal.SIGINT, sigint_handler)
-
-  if os.geteuid() != 0:
-    print_err('You must run this script as root')
-    sys.exit(1)
-
-  apply_help='After updating rule configuration, apply the idstools state.'
-
-  main_parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter)
-
-  subcommand_desc = textwrap.dedent(
-    """\
-      disabled            Manage and list disabled rules (add, remove, list)
-      enabled             Manage and list enabled rules (add, remove, list)
-      modify              Manage and list modified rules (add, remove, list)
-    """
-  )
-  subparsers = main_parser.add_subparsers(title='commands', description=subcommand_desc, metavar='', dest='command')
-
-
-  sid_or_regex_help = 'A valid SID (ex: "4321") or regular expression pattern (ex: "re:heartbleed|spectre")'
-
-  # Disabled actions
-  disabled = subparsers.add_parser('disabled')
-  disabled_sub = disabled.add_subparsers()
-
-  disabled_add = disabled_sub.add_parser('add')
-  disabled_add.set_defaults(func=add_rem_disabled)
-  disabled_add.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help)
-  disabled_add.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help)
-
-  disabled_rem = disabled_sub.add_parser('remove')
-  disabled_rem.set_defaults(func=add_rem_disabled, remove=True)
-  disabled_rem.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help)
-  disabled_rem.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help)
-
-  disabled_list = disabled_sub.add_parser('list')
-  disabled_list.set_defaults(func=list_disabled_rules)
-
-
-  # Enabled actions
-  enabled = subparsers.add_parser('enabled')
-  enabled_sub = enabled.add_subparsers()
-
-  enabled_add = enabled_sub.add_parser('add')
-  enabled_add.set_defaults(func=add_rem_enabled)
-  enabled_add.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help)
-  enabled_add.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help)
-
-  enabled_rem = enabled_sub.add_parser('remove')
-  enabled_rem.set_defaults(func=add_rem_enabled, remove=True)
-  enabled_rem.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help)
-  enabled_rem.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help)
-
-  enabled_list = enabled_sub.add_parser('list')
-  enabled_list.set_defaults(func=list_enabled_rules)
-  
-
-  search_term_help='A properly escaped regex search term (ex: "\\\$EXTERNAL_NET")'
-  replace_term_help='The text to replace the search term with'
-
-  # Modify actions
-  modify = subparsers.add_parser('modify')
-  modify_sub = modify.add_subparsers()
-
-  modify_add = modify_sub.add_parser('add')
-  modify_add.set_defaults(func=add_rem_modify)
-  modify_add.add_argument('sid_pattern', metavar='SID|REGEX', help=sid_or_regex_help)
-  modify_add.add_argument('search_term', metavar='SEARCH_TERM', help=search_term_help)
-  modify_add.add_argument('replace_term', metavar='REPLACE_TERM', help=replace_term_help)
-  modify_add.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help)
-
-  modify_rem = modify_sub.add_parser('remove')
-  modify_rem.set_defaults(func=add_rem_modify, remove=True)
-  modify_rem.add_argument('sid_pattern', metavar='SID', help=sid_or_regex_help)
-  modify_rem.add_argument('search_term', metavar='SEARCH_TERM', help=search_term_help)
-  modify_rem.add_argument('replace_term', metavar='REPLACE_TERM', help=replace_term_help)
-  modify_rem.add_argument('--apply', action='store_const', const=True, required=False, help=apply_help)
-
-  modify_list = modify_sub.add_parser('list')
-  modify_list.set_defaults(func=list_modified_rules)
-
-
-  # Begin parse + run
-  args = main_parser.parse_args(sys.argv[1:])
-  
-  if not hasattr(args, 'remove'):
-    args.remove = False
-
-  args.pillar = find_minion_pillar()
-
-  if hasattr(args, 'func'):
-    exit_code = args.func(args)
-  else:
-    if args.command is None:
-      main_parser.print_help()
-    else:
-      if args.command == 'disabled':
-        disabled.print_help()
-      elif args.command == 'enabled':
-        enabled.print_help()
-      elif args.command == 'modify':
-        modify.print_help()
-    sys.exit(0)
-  
-  sys.exit(exit_code)
-
-
-if __name__ == '__main__':
-  main()

salt/common/tools/sbin/so-rule-update

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-. /usr/sbin/so-common
-
-argstr=""
-for arg in "$@"; do
-    argstr="${argstr} \"${arg}\""
-done
-
-docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}"

salt/common/tools/sbin/so-soctopus-restart

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-restart soctopus $1

salt/common/tools/sbin/so-soctopus-start

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-start soctopus $1

salt/common/tools/sbin/so-soctopus-stop

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-stop soctopus $1

salt/common/tools/sbin/so-start

@@ -24,6 +24,7 @@ if [ $# -ge 1 ]; then
 	case $1 in
    		"all") salt-call state.highstate queue=True;;
    		"steno") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply pcap queue=True; fi ;; 
+   		"elastic-fleet") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply elasticfleet queue=True; fi ;; 
    		*) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; 
 	esac
 else

salt/common/tools/sbin/so-status

@@ -103,7 +103,7 @@ def output(options, console, code, data):
 def check_container_status(options, console):
   code = 0
   cli = "docker"
-  proc = subprocess.run([cli, 'ps', '--format', '{{json .}}'], stdout=subprocess.PIPE, encoding="utf-8")
+  proc = subprocess.run([cli, 'ps', '--format', 'json'], stdout=subprocess.PIPE, encoding="utf-8")
   if proc.returncode != 0:
     fail("Container system error; unable to obtain container process statuses")
 

salt/common/tools/sbin/so-test

@@ -5,4 +5,14 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+. /usr/sbin/so-common
+
+set -e
+
+# Playback live sample data onto monitor interface
 so-tcpreplay /opt/samples/* 2> /dev/null
+
+# Ingest sample pfsense log entry
+if is_sensor_node; then
+    echo "<134>$(date '+%b %d %H:%M:%S') filterlog[31624]: 84,,,1567509287,igb0.244,match,pass,in,4,0x0,,64,0,0,DF,6,tcp,64,192.168.1.1,10.10.10.10,56320,443,0,S,3333585167,,65535,,mss;nop;wscale;nop;nop;TS;sackOK;eol" | nc -uv -w1 127.0.0.1 514 > /dev/null 2>&1
+fi

salt/common/tools/sbin/so-zeek-logs

@@ -1,67 +0,0 @@
-#!/bin/bash
-local_salt_dir=/opt/so/saltstack/local
-
-zeek_logs_enabled() {
-  echo "zeeklogs:" > $local_salt_dir/pillar/zeeklogs.sls
-  echo "  enabled:" >> $local_salt_dir/pillar/zeeklogs.sls
-  for BLOG in "${BLOGS[@]}"; do
-    echo "    - $BLOG" | tr -d '"' >> $local_salt_dir/pillar/zeeklogs.sls
-  done
-}
-
-whiptail_manager_adv_service_zeeklogs() {
-  BLOGS=$(whiptail --title "so-zeek-logs" --checklist "Please Select Logs to Send:" 24 78 12 \
-  "conn" "Connection Logging" ON \
-  "dce_rpc" "RPC Logs" ON \
-  "dhcp" "DHCP Logs" ON \
-  "dnp3" "DNP3 Logs" ON \
-  "dns" "DNS Logs" ON \
-  "dpd" "DPD Logs" ON \
-  "files" "Files Logs" ON \
-  "ftp" "FTP Logs" ON \
-  "http" "HTTP Logs" ON \
-  "intel" "Intel Hits Logs" ON \
-  "irc" "IRC Chat Logs" ON \
-  "kerberos" "Kerberos Logs" ON \
-  "modbus" "MODBUS Logs" ON \
-  "notice" "Zeek Notice Logs" ON \
-  "ntlm" "NTLM Logs" ON \
-  "pe" "PE Logs" ON \
-  "radius" "Radius Logs" ON \
-  "rfb" "RFB Logs" ON \
-  "rdp" "RDP Logs" ON \
-  "sip" "SIP Logs" ON \
-  "smb_files" "SMB Files Logs" ON \
-  "smb_mapping" "SMB Mapping Logs" ON \
-  "smtp" "SMTP Logs" ON \
-  "snmp" "SNMP Logs" ON \
-  "ssh" "SSH Logs" ON \
-  "ssl" "SSL Logs" ON \
-  "syslog" "Syslog Logs" ON \
-  "tunnel" "Tunnel Logs" ON \
-  "weird" "Zeek Weird Logs" ON \
-  "mysql" "MySQL Logs" ON \
-  "socks" "SOCKS Logs" ON \
-  "x509" "x.509 Logs" ON 3>&1 1>&2 2>&3 )
-  
-  local exitstatus=$?
-
-  IFS=' ' read -ra BLOGS <<< "$BLOGS"
-
-  return $exitstatus
-}
-
-whiptail_manager_adv_service_zeeklogs
-return_code=$?
-case $return_code in
-  1)
-    whiptail --title "so-zeek-logs" --msgbox "Cancelling. No changes have been made." 8 75
-    ;;
-  255)
-    whiptail --title "so-zeek-logs" --msgbox "Whiptail error occured, exiting." 8 75
-    ;;
-  *)
-    zeek_logs_enabled
-    ;;
-esac
-

salt/common/tools/sbin_jinja/so-desktop-install

@@ -5,18 +5,18 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+source /usr/sbin/so-common
+doc_desktop_url="$DOC_BASE_URL/desktop.html"
 
-{# we only want the script to install the workstation if it is Rocky -#}
-{% if grains.os == 'Rocky' -%}
+{# we only want the script to install the desktop if it is OEL -#}
+{% if grains.os == 'OEL' -%}
 {#   if this is a manager -#}
 {%   if grains.master == grains.id.split('_')|first -%}
 
-source /usr/sbin/so-common
-doc_workstation_url="$DOC_BASE_URL/analyst-vm.html"
-pillar_file="/opt/so/saltstack/local/pillar/minions/{{grains.id}}.sls"
+pillar_file="/opt/so/saltstack/local/pillar/minions/adv_{{grains.id}}.sls"
 
 if [ -f "$pillar_file" ]; then
-  if ! grep -q "^workstation:$" "$pillar_file"; then
+  if ! grep -q "^desktop:$" "$pillar_file"; then
 
     FIRSTPASS=yes
     while [[ $INSTALL != "yes" ]] && [[ $INSTALL != "no" ]]; do
@@ -26,7 +26,7 @@ if [ -f "$pillar_file" ]; then
         echo "##    _______________________________    ##"
         echo "##                                       ##"
         echo "##    Installing the Security Onion      ##"
-        echo "##   analyst node on this device will    ##"
+        echo "##     Desktop on this device will       ##"
         echo "##       make permanent changes to       ##"
         echo "##              the system.              ##"
         echo "##    A system reboot will be required   ##"
@@ -42,50 +42,55 @@ if [ -f "$pillar_file" ]; then
     done
 
     if [[ $INSTALL == "no" ]]; then
-      echo "Exiting analyst node installation."
+      echo "Exiting desktop node installation."
       exit 0
     fi
 
-    # Add workstation pillar to the minion's pillar file
+    # Add desktop pillar to the minion's pillar file
     printf '%s\n'\
-      "workstation:"\
+      "desktop:"\
       "  gui:"\
       "    enabled: true"\
 		  "" >> "$pillar_file"
-    echo "Applying the workstation state. This could take some time since there are many packages that need to be installed."
-    if salt-call state.apply workstation -linfo queue=True; then # make sure the state ran successfully
+    echo "Applying the desktop state. This could take some time since there are many packages that need to be installed."
+    if salt-call state.apply desktop -linfo queue=True; then # make sure the state ran successfully
       echo ""
-      echo "Analyst workstation has been installed!"
+      echo "Security Onion Desktop has been installed!"
       echo "Press ENTER to reboot or Ctrl-C to cancel."
       read pause
 
       reboot;
     else
-      echo "There was an issue applying the workstation state. Please review the log above or at /opt/so/log/salt/minion."
+      echo "There was an issue applying the desktop state. Please review the log above or at /opt/so/log/salt/minion."
     fi
-  else # workstation is already added
-    echo "The workstation pillar already exists in $pillar_file."
-    echo "To enable/disable the gui, set 'workstation:gui:enabled' to true or false in $pillar_file."
-    echo "Additional documentation can be found at $doc_workstation_url."
+  else # desktop is already added
+    echo "The desktop pillar already exists in $pillar_file."
+    echo "To enable/disable the gui, set 'desktop:gui:enabled' to true or false in $pillar_file. Alternatively, this can be set in the SOC UI under advanced."
+    echo "Additional documentation can be found at $doc_desktop_url."
   fi
 else # if the pillar file doesn't exist
-  echo "Could not find $pillar_file and add the workstation pillar."
+  echo "Could not find $pillar_file and add the desktop pillar."
 fi
 
 {#-  if this is not a manager #}
 {%   else -%}
 
-echo "Since this is not a manager, the pillar values to enable analyst workstation must be set manually. Please view the documentation at $doc_workstation_url."
+echo "Since this is not a manager, the pillar values to enable Security Onion Desktop must be set manually. This can be enabled in the SOC UI under advanced by adding the following:"
+echo "desktop:"
+echo "  gui:"
+echo "    enabled: true"
+echo ""
+echo "Please view the documentation at $doc_desktop_url."
 
 {#- endif if this is a manager #}
 {%   endif -%}
 
-{#- if not Rocky #}
+{#- if not OEL #}
 {%- else %}
 
-echo "The Analyst Workstation can only be installed on Rocky. Please view the documentation at $doc_workstation_url."
+echo "The Security Onion Desktop can only be installed on Oracle Linux. Please view the documentation at $doc_desktop_url."
 
-{#- endif grains.os == Rocky #}
+{#- endif grains.os == OEL #}
 {% endif -%}
 
 exit 0

salt/common/tools/sbin_jinja/so-import-evtx

@@ -0,0 +1,245 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{%- set MANAGER = salt['grains.get']('master') %}
+{%- set VERSION = salt['pillar.get']('global:soversion') %}
+{%- set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
+{%- set MANAGERIP = salt['pillar.get']('global:managerip') %}
+{%- set URLBASE = salt['pillar.get']('global:url_base') %}
+{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
+{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
+
+INDEX_DATE=$(date +'%Y.%m.%d')
+LOG_FILE=/nsm/import/evtx-import.log
+
+. /usr/sbin/so-common
+
+function usage {
+  cat << EOF
+Usage: $0 [options] <evtx-file-1> [evtx-file-2] [evtx-file-*]
+
+Imports one or more evtx files into Security Onion. The evtx files will be analyzed and made available for review in the Security Onion toolset.
+
+Options:
+  --json      Outputs summary in JSON format. Implies --quiet.
+  --quiet     Silences progress information to stdout.
+  --shift     Adds a time shift. Accepts a single argument that is intended to be the date of the last record, and shifts the dates of the previous records accordingly.
+              Ex. sudo so-import-evtx --shift "2023-08-01 01:01:01" example.evtx 
+EOF
+}
+
+quiet=0
+json=0
+INPUT_FILES=
+while [[ $# -gt 0 ]]; do
+  param=$1
+  shift
+  case "$param" in
+    --json)
+      json=1
+      quiet=1
+      ;;
+    --quiet)
+      quiet=1
+      ;;
+    --shift)
+      SHIFTDATE=$1
+      shift
+      ;;
+    -*) 
+      echo "Encountered unexpected parameter: $param"
+      usage
+      exit 1
+      ;;
+    *) 
+      if [[ "$INPUT_FILES" != "" ]]; then
+        INPUT_FILES="$INPUT_FILES $param"
+      else
+        INPUT_FILES="$param"
+      fi
+      ;;
+  esac
+done
+
+function status {
+  msg=$1
+  [[ $quiet -eq 1 ]] && return
+  echo "$msg"
+}
+
+function evtx2es() {
+  EVTX=$1
+  HASH=$2
+  SHIFTDATE=$3
+  
+  docker run --rm \
+    -e "SHIFTTS=$SHIFTDATE" \
+    -v "$EVTX:/tmp/data.evtx" \
+    -v "/nsm/import/$HASH/evtx/:/tmp/evtx/" \
+    -v "/nsm/import/$HASH/evtx-end_newest:/tmp/newest" \
+    -v "/nsm/import/$HASH/evtx-start_oldest:/tmp/oldest" \
+    --entrypoint "/evtx_calc_timestamps.sh" \
+    {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} >> $LOG_FILE 2>&1
+}
+
+# if no parameters supplied, display usage
+if [ "$INPUT_FILES" == "" ]; then
+  usage
+  exit 1
+fi
+
+# ensure this is a Manager node
+require_manager @> /dev/null
+
+# verify that all parameters are files
+for i in $INPUT_FILES; do
+  if ! [ -f "$i" ]; then
+    echo "\"$i\" is not a valid file!"
+    exit 2
+  fi
+done
+
+# track oldest start and newest end so that we can generate the Kibana search hyperlink at the end
+START_OLDEST="2050-12-31"
+END_NEWEST="1971-01-01"
+
+INVALID_EVTXS_COUNT=0
+VALID_EVTXS_COUNT=0
+SKIPPED_EVTXS_COUNT=0
+
+# paths must be quoted in case they include spaces
+for EVTX in $INPUT_FILES; do
+  EVTX=$(/usr/bin/realpath "$EVTX")
+  status "Processing Import: ${EVTX}"
+  if ! [ -z "$SHIFTDATE" ]; then
+    status "- timeshifting logs to end date of $SHIFTDATE"
+  fi
+  # generate a unique hash to assist with dedupe checks
+  HASH=$(md5sum "${EVTX}" | awk '{ print $1 }')
+  HASH_DIR=/nsm/import/${HASH}
+  status "- assigning unique identifier to import: $HASH"
+
+  if [[ "$HASH_FILTERS" == "" ]]; then
+    HASH_FILTERS="import.id:${HASH}"
+    HASHES="${HASH}"
+  else
+    HASH_FILTERS="$HASH_FILTERS%20OR%20import.id:${HASH}"
+    HASHES="${HASHES} ${HASH}"
+  fi
+
+  if [ -d $HASH_DIR ]; then
+    status "- this EVTX has already been imported; skipping"
+    SKIPPED_EVTXS_COUNT=$((SKIPPED_EVTXS_COUNT + 1))
+  else
+    # create EVTX directory
+    EVTX_DIR=$HASH_DIR/evtx
+    mkdir -p $EVTX_DIR
+    # create import timestamp files
+    for i in evtx-start_oldest evtx-end_newest; do
+      if ! [ -f "$i" ]; then
+        touch /nsm/import/$HASH/$i
+      fi
+    done
+
+    # import evtx and write them to import ingest pipeline
+    status "- importing logs to Elasticsearch..."
+    evtx2es "${EVTX}" $HASH "$SHIFTDATE"
+    if [[ $? -ne 0 ]]; then
+      INVALID_EVTXS_COUNT=$((INVALID_EVTXS_COUNT + 1))
+      status "- WARNING: This evtx file may not have fully imported successfully"
+    else
+      VALID_EVTXS_COUNT=$((VALID_EVTXS_COUNT + 1))
+    fi
+
+    cp -f "${EVTX}" "${EVTX_DIR}"/data.evtx
+    chmod 644 "${EVTX_DIR}"/data.evtx
+
+  fi # end of valid evtx
+
+  # determine start and end and make sure they aren't reversed
+  START=$(cat /nsm/import/$HASH/evtx-start_oldest)
+  END=$(cat /nsm/import/$HASH/evtx-end_newest)
+  START_EPOCH=`date -d "$START" +"%s"`
+  END_EPOCH=`date -d "$END" +"%s"`
+  if [ "$START_EPOCH" -gt "$END_EPOCH" ]; then
+    TEMP=$START
+    START=$END
+    END=$TEMP
+  fi
+
+  # compare $START to $START_OLDEST
+  START_COMPARE=$(date -d $START +%s)
+  START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s)
+  if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then
+    START_OLDEST=$START
+  fi
+
+  # compare $ENDNEXT to $END_NEWEST
+  ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"`
+  ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s)
+  END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s)
+  if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then
+    END_NEWEST=$ENDNEXT
+  fi
+
+  status
+
+done # end of for-loop processing evtx files
+
+# output final messages
+if [[ $INVALID_EVTXS_COUNT -gt 0 ]]; then
+  status
+  status "Please note!  One or more evtx was invalid!  You can scroll up to see which ones were invalid."
+fi
+
+START_OLDEST_FORMATTED=`date +%Y-%m-%d --date="$START_OLDEST"`
+START_OLDEST_SLASH=$(echo $START_OLDEST_FORMATTED | sed -e 's/-/%2F/g')
+END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
+
+if [[ $VALID_EVTXS_COUNT -gt 0 ]] || [[ $SKIPPED_EVTXS_COUNT -gt 0 ]]; then
+  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20-sankey%20event.dataset%20event.category%2a%20%7C%20groupby%20-pie%20event.category%20%7C%20groupby%20-bar%20event.module%20%7C%20groupby%20event.dataset%20%7C%20groupby%20event.module%20%7C%20groupby%20event.category%20%7C%20groupby%20observer.name%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
+
+  status "Import complete!"
+  status
+  status "Use the following hyperlink to view the imported data. Triple-click to quickly highlight the entire hyperlink and then copy it into a browser:"
+  status
+  status "$URL"
+  status
+  status "or, manually set the Time Range to be (in UTC):"
+  status
+  status "From: $START_OLDEST_FORMATTED    To: $END_NEWEST"
+  status
+  status "Note: It can take 30 seconds or more for events to appear in Security Onion Console."
+  RESULT=0
+else
+  START_OLDEST=
+  END_NEWEST=
+  URL=
+  RESULT=1
+fi
+
+if [[ $json -eq 1 ]]; then
+  jq -n \
+    --arg success_count "$VALID_EVTXS_COUNT" \
+    --arg fail_count "$INVALID_EVTXS_COUNT" \
+    --arg skipped_count "$SKIPPED_EVTXS_COUNT" \
+    --arg begin_date "$START_OLDEST" \
+    --arg end_date "$END_NEWEST" \
+    --arg url "$URL" \
+    --arg hashes "$HASHES" \
+    '''{
+      success_count: $success_count,
+      fail_count: $fail_count,
+      skipped_count: $skipped_count,
+      begin_date: $begin_date,
+      end_date: $end_date,
+      url: $url,
+      hash: ($hashes / " ")
+    }'''
+fi
+
+exit $RESULT

salt/common/tools/sbin_jinja/so-import-pcap

@@ -15,12 +15,51 @@
 
 function usage {
   cat << EOF
-Usage: $0 <pcap-file-1> [pcap-file-2] [pcap-file-N]
+Usage: $0 [options] <pcap-file-1> [pcap-file-2] [pcap-file-N]
 
 Imports one or more PCAP files onto a sensor node. The PCAP traffic will be analyzed and made available for review in the Security Onion toolset.
+
+Options:
+  --json      Outputs summary in JSON format. Implies --quiet.
+  --quiet     Silences progress information to stdout.
 EOF
 }
 
+quiet=0
+json=0
+INPUT_FILES=
+while [[ $# -gt 0 ]]; do
+  param=$1
+  shift
+  case "$param" in
+    --json)
+      json=1
+      quiet=1
+      ;;
+    --quiet)
+      quiet=1
+      ;;
+    -*) 
+      echo "Encountered unexpected parameter: $param"
+      usage
+      exit 1
+      ;;
+    *) 
+      if [[ "$INPUT_FILES" != "" ]]; then
+        INPUT_FILES="$INPUT_FILES $param"
+      else
+        INPUT_FILES="$param"
+      fi
+      ;;
+  esac
+done
+
+function status {
+  msg=$1
+  [[ $quiet -eq 1 ]] && return
+  echo "$msg"
+}
+
 function pcapinfo() {
   PCAP=$1
   ARGS=$2
@@ -50,6 +89,7 @@ function suricata() {
     -v ${LOG_PATH}:/var/log/suricata/:rw \
     -v ${NSM_PATH}/:/nsm/:rw \
     -v "$PCAP:/input.pcap:ro" \
+    -v /dev/null:/nsm/suripcap:rw \
     -v /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro \
     {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-suricata:{{ VERSION }} \
     --runmode single -k none -r /input.pcap > $LOG_PATH/console.log 2>&1
@@ -84,7 +124,7 @@ function zeek() {
 }
 
 # if no parameters supplied, display usage
-if [ $# -eq 0 ]; then
+if [ "$INPUT_FILES" == "" ]; then
   usage
   exit 1
 fi
@@ -96,31 +136,30 @@ if [ ! -d /opt/so/conf/suricata ]; then
 fi
 
 # verify that all parameters are files
-for i in "$@"; do
+for i in $INPUT_FILES; do
   if ! [ -f "$i" ]; then
-    usage
     echo "\"$i\" is not a valid file!"
     exit 2
   fi
 done
 
-# track if we have any valid or invalid pcaps
-INVALID_PCAPS="no"
-VALID_PCAPS="no"
-
 # track oldest start and newest end so that we can generate the Kibana search hyperlink at the end
 START_OLDEST="2050-12-31"
 END_NEWEST="1971-01-01"
 
+INVALID_PCAPS_COUNT=0
+VALID_PCAPS_COUNT=0
+SKIPPED_PCAPS_COUNT=0
+
 # paths must be quoted in case they include spaces
-for PCAP in "$@"; do
+for PCAP in $INPUT_FILES; do
   PCAP=$(/usr/bin/realpath "$PCAP")
-  echo "Processing Import: ${PCAP}"
-  echo "- verifying file"
+  status "Processing Import: ${PCAP}"
+  status "- verifying file"
   if ! pcapinfo "${PCAP}" > /dev/null 2>&1; then
     # try to fix pcap and then process the fixed pcap directly
     PCAP_FIXED=`mktemp /tmp/so-import-pcap-XXXXXXXXXX.pcap`
-    echo "- attempting to recover corrupted PCAP file"
+    status "- attempting to recover corrupted PCAP file"
     pcapfix "${PCAP}" "${PCAP_FIXED}"
     # Make fixed file world readable since the Suricata docker container will runas a non-root user
     chmod a+r "${PCAP_FIXED}"
@@ -131,33 +170,44 @@ for PCAP in "$@"; do
   # generate a unique hash to assist with dedupe checks
   HASH=$(md5sum "${PCAP}" | awk '{ print $1 }')
   HASH_DIR=/nsm/import/${HASH}
-  echo "- assigning unique identifier to import: $HASH"
+  status "- assigning unique identifier to import: $HASH"
 
-  if [ -d $HASH_DIR ]; then
-    echo "- this PCAP has already been imported; skipping"
-    INVALID_PCAPS="yes"
-  elif pcapinfo "${PCAP}" |egrep -q "Last packet time:    1970-01-01|Last packet time:    n/a"; then
-    echo "- this PCAP file is invalid; skipping"
-    INVALID_PCAPS="yes"
+  pcap_data=$(pcapinfo "${PCAP}")
+  if ! echo "$pcap_data" | grep -q "First packet time:" || echo "$pcap_data" |egrep -q "Last packet time:    1970-01-01|Last packet time:    n/a"; then
+    status "- this PCAP file is invalid; skipping"
+    INVALID_PCAPS_COUNT=$((INVALID_PCAPS_COUNT + 1))
   else
-    VALID_PCAPS="yes"
+    if [ -d $HASH_DIR ]; then
+      status "- this PCAP has already been imported; skipping"
+      SKIPPED_PCAPS_COUNT=$((SKIPPED_PCAPS_COUNT + 1))
+    else
+      VALID_PCAPS_COUNT=$((VALID_PCAPS_COUNT + 1))
 
-    PCAP_DIR=$HASH_DIR/pcap
-    mkdir -p $PCAP_DIR
+      PCAP_DIR=$HASH_DIR/pcap
+      mkdir -p $PCAP_DIR
 
-    # generate IDS alerts and write them to standard pipeline
-    echo "- analyzing traffic with Suricata"
-    suricata "${PCAP}" $HASH
-    {% if salt['pillar.get']('global:mdengine') == 'ZEEK' %}
-    # generate Zeek logs and write them to a unique subdirectory in /nsm/import/zeek/
-    # since each run writes to a unique subdirectory, there is no need for a lock file
-    echo "- analyzing traffic with Zeek"
-    zeek "${PCAP}" $HASH
-    {% endif %}
+      # generate IDS alerts and write them to standard pipeline
+      status "- analyzing traffic with Suricata"
+      suricata "${PCAP}" $HASH
+      {% if salt['pillar.get']('global:mdengine') == 'ZEEK' %}
+      # generate Zeek logs and write them to a unique subdirectory in /nsm/import/zeek/
+      # since each run writes to a unique subdirectory, there is no need for a lock file
+      status "- analyzing traffic with Zeek"
+      zeek "${PCAP}" $HASH
+      {% endif %}
+    fi
+
+    if [[ "$HASH_FILTERS" == "" ]]; then
+      HASH_FILTERS="import.id:${HASH}"
+      HASHES="${HASH}"
+    else
+      HASH_FILTERS="$HASH_FILTERS%20OR%20import.id:${HASH}"
+      HASHES="${HASHES} ${HASH}"
+    fi
 
     START=$(pcapinfo "${PCAP}" -a |grep "First packet time:" | awk '{print $4}')
     END=$(pcapinfo "${PCAP}" -e |grep "Last packet time:" | awk '{print $4}')
-    echo "- saving PCAP data spanning dates $START through $END"
+    status "- found PCAP data spanning dates $START through $END"
 
     # compare $START to $START_OLDEST
     START_COMPARE=$(date -d $START +%s)
@@ -179,37 +229,62 @@ for PCAP in "$@"; do
 
   fi # end of valid pcap
 
-  echo
+  status
 
 done # end of for-loop processing pcap files
 
 # remove temp files
-echo "Cleaning up:"
 for TEMP_PCAP in ${TEMP_PCAPS[@]}; do
-  echo "- removing temporary pcap $TEMP_PCAP"
+  status "- removing temporary pcap $TEMP_PCAP"
   rm -f $TEMP_PCAP
 done
 
 # output final messages
-if [ "$INVALID_PCAPS" = "yes" ]; then
-  echo
-  echo "Please note!  One or more pcaps was invalid!  You can scroll up to see which ones were invalid."
+if [[ $INVALID_PCAPS_COUNT -gt 0 ]]; then
+  status
+  status "WARNING: One or more pcaps was invalid. Scroll up to see which ones were invalid."
 fi
 
 START_OLDEST_SLASH=$(echo $START_OLDEST | sed -e 's/-/%2F/g')
 END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
+if [[ $VALID_PCAPS_COUNT -gt 0 ]] || [[ $SKIPPED_PCAPS_COUNT -gt 0 ]]; then
+  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20event.module*%20%7C%20groupby%20-sankey%20event.module*%20event.dataset%20%7C%20groupby%20event.dataset%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port%20%7C%20groupby%20network.protocol%20%7C%20groupby%20rule.name%20rule.category%20event.severity_label%20%7C%20groupby%20dns.query.name%20%7C%20groupby%20file.mime_type%20%7C%20groupby%20http.virtual_host%20http.uri%20%7C%20groupby%20notice.note%20notice.message%20notice.sub_message%20%7C%20groupby%20ssl.server_name%20%7C%20groupby%20source_geo.organization_name%20source.geo.country_name%20%7C%20groupby%20destination_geo.organization_name%20destination.geo.country_name&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
 
-if [ "$VALID_PCAPS" = "yes" ]; then
-cat << EOF
-
-Import complete!
-
-You can use the following hyperlink to view data in the time range of your import.  You can triple-click to quickly highlight the entire hyperlink and you can then copy it into your browser:
-https://{{ URLBASE }}/#/dashboards?q=import.id:${HASH}%20%7C%20groupby%20-sankey%20event.dataset%20event.category%2a%20%7C%20groupby%20-pie%20event.category%20%7C%20groupby%20-bar%20event.module%20%7C%20groupby%20event.dataset%20%7C%20groupby%20event.module%20%7C%20groupby%20event.category%20%7C%20groupby%20observer.name%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC
-
-or you can manually set your Time Range to be (in UTC):
-From: $START_OLDEST    To: $END_NEWEST
-
-Please note that it may take 30 seconds or more for events to appear in Security Onion Console.
-EOF
+  status "Import complete!"
+  status
+  status "Use the following hyperlink to view the imported data. Triple-click to quickly highlight the entire hyperlink and then copy it into a browser:"
+  status "$URL"
+  status
+  status "or, manually set the Time Range to be (in UTC):"
+  status "From: $START_OLDEST    To: $END_NEWEST"
+  status
+  status "Note: It can take 30 seconds or more for events to appear in Security Onion Console."
+  RESULT=0
+else
+  START_OLDEST=
+  END_NEWEST=
+  URL=
+  RESULT=1
 fi
+
+if [[ $json -eq 1 ]]; then
+  jq -n \
+    --arg success_count "$VALID_PCAPS_COUNT" \
+    --arg fail_count "$INVALID_PCAPS_COUNT" \
+    --arg skipped_count "$SKIPPED_PCAPS_COUNT" \
+    --arg begin_date "$START_OLDEST" \
+    --arg end_date "$END_NEWEST" \
+    --arg url "$URL" \
+    --arg hashes "$HASHES" \
+    '''{
+      success_count: $success_count,
+      fail_count: $fail_count,
+      skipped_count: $skipped_count,
+      begin_date: $begin_date,
+      end_date: $end_date,
+      url: $url,
+      hash: ($hashes / " ")
+    }'''
+fi
+
+exit $RESULT

salt/common/tools/sbin_jinja/so-raid-status

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+. /usr/sbin/so-common
+
+{%- if salt['grains.get']('sosmodel', '') %}
+{%- set model = salt['grains.get']('sosmodel') %}
+model={{ model }}
+# Don't need cloud images to use this
+if [[ $model =~ ^(SO2AMI01|SO2AZI01|SO2GCI01)$ ]]; then
+    exit 0
+fi
+{%- else %}
+echo "This is not an appliance"
+exit 0
+{%- endif %}
+if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200|SOSSNNV|SOSMN)$ ]]; then
+	is_bossraid=true
+fi
+if [[ $model =~ ^(SOSSNNV|SOSMN)$ ]]; then
+	is_swraid=true
+fi
+if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200)$ ]]; then
+	is_hwraid=true
+fi
+
+check_nsm_raid() {
+  PERCCLI=$(/opt/raidtools/perccli/perccli64 /c0/v0 show|grep RAID|grep Optl)
+  MEGACTL=$(/opt/raidtools/megasasctl |grep optimal)
+
+  if [[ $APPLIANCE == '1' ]]; then
+    if [[ -n $PERCCLI ]]; then
+      HWRAID=0
+    elif [[ -n $MEGACTL ]]; then
+      HWRAID=0
+    else
+      HWRAID=1
+    fi
+
+  fi
+
+}
+
+check_boss_raid() {
+  MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional)
+  MVTEST=$(/usr/local/bin/mvcli info -o vd | grep "No adapter")
+
+  # Check to see if this is a SM based system
+  if [[ -z $MVTEST ]]; then
+    if [[ -n $MVCLI ]]; then
+      BOSSRAID=0
+    else
+      BOSSRAID=1
+    fi
+  else
+    # This doesn't have boss raid so lets make it 0
+    BOSSRAID=0
+  fi
+}
+
+check_software_raid() {
+  SWRC=$(grep "_" /proc/mdstat)
+  if [[ -n $SWRC ]]; then
+      # RAID is failed in some way
+      SWRAID=1
+  else
+      SWRAID=0
+  fi
+}
+
+# Set everything to 0
+SWRAID=0
+BOSSRAID=0
+HWRAID=0
+
+if [[ $is_hwraid ]]; then
+	check_nsm_raid
+fi
+if [[ $is_bossraid ]]; then
+	check_boss_raid
+fi
+if [[ $is_swraid ]]; then
+	check_software_raid
+fi
+
+sum=$(($SWRAID + $BOSSRAID + $HWRAID))
+
+if [[ $sum == "0" ]]; then
+  RAIDSTATUS=0
+else
+  RAIDSTATUS=1
+fi
+
+echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log