Merge pull request #10970 from Security-Onion-Solutions/2.4/dev

2.4.5 RC2

DOWNLOAD_AND_VERIFY_ISO.md

@@ -1,47 +1,47 @@
-### 2.3.120-20220425 ISO image built on 2022/04/25
+### 2.4.5-20230807 ISO image released on 2023/08/07
 
 
 
 ### Download and Verify
 
-2.3.120-20220425 ISO image:  
+2.4.5-20230807 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.120-20220425.iso
+https://download.securityonion.net/file/securityonion/securityonion-2.4.5-20230807.iso
-
+ 
-MD5: C99729E452B064C471BEF04532F28556  
+MD5: F83FD635025A3A65B380EAFCEB61A92E  
-SHA1: 60BF07D5347C24568C7B793BFA9792E98479CFBF  
+SHA1: 5864D4CD520617E3328A3D956CAFCC378A8D2D08  
-SHA256: CD17D0D7CABE21D45FA45E1CF91C5F24EB9608C79FF88480134E5592AFDD696E 
+SHA256: D333BAE0DD198DFD80DF59375456D228A4E18A24EDCDB15852CD4CA3F92B69A7  
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.120-20220425.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.5-20230807.iso.sig
 
 Signing key:  
-https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
+https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS  
 
 For example, here are the steps you can use on most Linux distributions to download and verify our Security Onion ISO image.
 
 Download and import the signing key:  
 ```
-wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS -O - | gpg --import -  
+wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS -O - | gpg --import -  
 ```
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.120-20220425.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.5-20230807.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.120-20220425.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.4.5-20230807.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.120-20220425.iso.sig securityonion-2.3.120-20220425.iso
+gpg --verify securityonion-2.4.5-20230807.iso.sig securityonion-2.4.5-20230807.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 25 Apr 2022 08:20:40 AM EDT using RSA key ID FE507013
+gpg: Signature made Sat 05 Aug 2023 10:12:46 AM EDT using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.
@@ -49,4 +49,4 @@ Primary key fingerprint: C804 A93D 36BE 0C73 3EA1  9644 7C10 60B7 FE50 7013
 ```
 
 Once you've verified the ISO image, you're ready to proceed to our Installation guide:  
-https://docs.securityonion.net/en/2.3/installation.html
+https://docs.securityonion.net/en/2.4/installation.html

README.md

@@ -1,20 +1,26 @@
-## Security Onion 2.4
+## Security Onion 2.4 Release Candidate 2 (RC2)
 
-Security Onion 2.4 is here!
+Security Onion 2.4 Release Candidate 2 (RC2) is here!
 
 ## Screenshots
 
 Alerts
-![Alerts](./assets/images/screenshots/alerts.png)
+![Alerts](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/50_alerts.png)
 
 Dashboards
-![Dashboards](./assets/images/screenshots/dashboards.png)
+![Dashboards](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/51_dashboards.png)
 
 Hunt
-![Hunt](./assets/images/screenshots/hunt.png)
+![Hunt](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/52_hunt.png)
 
-Cases
+PCAP
-![Cases](./assets/images/screenshots/cases-comments.png)
+![PCAP](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/53_pcap.png)
+
+Grid
+![Grid](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/57_grid.png)
+
+Config
+![Config](https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion-docs/2.4/images/61_config.png)
 
 ### Release Notes
 

VERSION

@@ -1 +1 @@
-2.4.0
+2.4.5

pillar/logrotate/init.sls

@@ -1,13 +0,0 @@
-logrotate:
-  conf: | 
-    daily
-    rotate 14
-    missingok
-    copytruncate
-    compress
-    create
-    extension .log
-    dateext
-    dateyesterday
-  group_conf: |
-    su root socore

pillar/logstash/helix.sls

@@ -1,42 +0,0 @@
-logstash:
-  pipelines:
-    helix:
-      config:
-        - so/0010_input_hhbeats.conf
-        - so/1033_preprocess_snort.conf
-        - so/1100_preprocess_bro_conn.conf
-        - so/1101_preprocess_bro_dhcp.conf
-        - so/1102_preprocess_bro_dns.conf
-        - so/1103_preprocess_bro_dpd.conf
-        - so/1104_preprocess_bro_files.conf
-        - so/1105_preprocess_bro_ftp.conf
-        - so/1106_preprocess_bro_http.conf
-        - so/1107_preprocess_bro_irc.conf
-        - so/1108_preprocess_bro_kerberos.conf
-        - so/1109_preprocess_bro_notice.conf
-        - so/1110_preprocess_bro_rdp.conf
-        - so/1111_preprocess_bro_signatures.conf
-        - so/1112_preprocess_bro_smtp.conf
-        - so/1113_preprocess_bro_snmp.conf
-        - so/1114_preprocess_bro_software.conf
-        - so/1115_preprocess_bro_ssh.conf
-        - so/1116_preprocess_bro_ssl.conf
-        - so/1117_preprocess_bro_syslog.conf
-        - so/1118_preprocess_bro_tunnel.conf
-        - so/1119_preprocess_bro_weird.conf
-        - so/1121_preprocess_bro_mysql.conf
-        - so/1122_preprocess_bro_socks.conf
-        - so/1123_preprocess_bro_x509.conf
-        - so/1124_preprocess_bro_intel.conf
-        - so/1125_preprocess_bro_modbus.conf
-        - so/1126_preprocess_bro_sip.conf
-        - so/1127_preprocess_bro_radius.conf
-        - so/1128_preprocess_bro_pe.conf
-        - so/1129_preprocess_bro_rfb.conf
-        - so/1130_preprocess_bro_dnp3.conf
-        - so/1131_preprocess_bro_smb_files.conf
-        - so/1132_preprocess_bro_smb_mapping.conf
-        - so/1133_preprocess_bro_ntlm.conf
-        - so/1134_preprocess_bro_dce_rpc.conf
-        - so/8001_postprocess_common_ip_augmentation.conf
-        - so/9997_output_helix.conf.jinja

pillar/logstash/init.sls

@@ -4,6 +4,7 @@ logstash:
       - 0.0.0.0:3765:3765
       - 0.0.0.0:5044:5044
       - 0.0.0.0:5055:5055
+      - 0.0.0.0:5056:5056
       - 0.0.0.0:5644:5644
       - 0.0.0.0:6050:6050
       - 0.0.0.0:6051:6051

pillar/logstash/manager.sls

@@ -1,8 +0,0 @@
-logstash:
-  pipelines:
-    manager:
-      config:
-        - so/0011_input_endgame.conf
-        - so/0012_input_elastic_agent.conf     
-        - so/9999_output_redis.conf.jinja
-        

pillar/logstash/nodes.sls

@@ -2,7 +2,7 @@
 {% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %}
 {% for minionid, ip in salt.saltutil.runner(
     'mine.get',
-    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-helix ',
+    tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ',
     fun='network.ip_addrs',
     tgt_type='compound') | dictsort()
 %}

pillar/logstash/receiver.sls

@@ -1,8 +0,0 @@
-logstash:
-  pipelines:
-    receiver:
-      config:
-        - so/0011_input_endgame.conf
-        - so/0012_input_elastic_agent.conf
-        - so/9999_output_redis.conf.jinja
-        

pillar/logstash/search.sls

@@ -1,7 +0,0 @@
-logstash:
-  pipelines:
-    search:
-      config:
-        - so/0900_input_redis.conf.jinja
-        - so/9805_output_elastic_agent.conf.jinja
-        - so/9900_output_endgame.conf.jinja

pillar/soc/license.sls

@@ -0,0 +1,14 @@
+# Copyright Jason Ertel (github.com/jertel).
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with 
+# the Elastic License 2.0.
+
+# Note: Per the Elastic License 2.0, the second limitation states:
+#
+#   "You may not move, change, disable, or circumvent the license key functionality
+#    in the software, and you may not remove or obscure any functionality in the
+#    software that is protected by the license key."
+
+# This file is generated by Security Onion and contains a list of license-enabled features. 
+features: []

pillar/top.sls

@@ -1,44 +1,26 @@
 base:
   '*':
-    - patch.needs_restarting
+    - global.soc_global
-    - ntp.soc_ntp
+    - global.adv_global
-    - ntp.adv_ntp
-    - logrotate
     - docker.soc_docker
     - docker.adv_docker
+    - firewall.soc_firewall
+    - firewall.adv_firewall
+    - influxdb.token
+    - logrotate.soc_logrotate
+    - logrotate.adv_logrotate
+    - nginx.soc_nginx
+    - nginx.adv_nginx
+    - node_data.ips
+    - ntp.soc_ntp
+    - ntp.adv_ntp
+    - patch.needs_restarting
+    - patch.soc_patch
+    - patch.adv_patch
     - sensoroni.soc_sensoroni
     - sensoroni.adv_sensoroni
     - telegraf.soc_telegraf
     - telegraf.adv_telegraf
-    - influxdb.token
-    - node_data.ips
-
-  '* and not *_eval and not *_import':
-    - logstash.nodes
-
-  '*_eval or *_heavynode or *_sensor or *_standalone or *_import':
-    - match: compound
-    - zeek
-    - bpf.soc_bpf
-    - bpf.adv_bpf
-
-  '*_managersearch or *_heavynode':
-    - match: compound
-    - logstash
-    - logstash.manager
-    - logstash.search
-    - logstash.soc_logstash
-    - logstash.adv_logstash
-    - elasticsearch.index_templates
-    - elasticsearch.soc_elasticsearch
-    - elasticsearch.adv_elasticsearch
-
-  '*_manager':
-    - logstash
-    - logstash.manager
-    - logstash.soc_logstash
-    - logstash.adv_logstash
-    - elasticsearch.index_templates
 
   '*_manager or *_managersearch':
     - match: compound
@@ -49,14 +31,20 @@ base:
     - kibana.secrets
     {% endif %}
     - secrets
-    - soc_global
-    - adv_global
     - manager.soc_manager
     - manager.adv_manager
     - idstools.soc_idstools
     - idstools.adv_idstools
+    - logstash.nodes
+    - logstash.soc_logstash
+    - logstash.adv_logstash
     - soc.soc_soc
     - soc.adv_soc
+    - soc.license
+    - soctopus.soc_soctopus
+    - soctopus.adv_soctopus
+    - kibana.soc_kibana
+    - kibana.adv_kibana
     - kratos.soc_kratos
     - kratos.adv_kratos
     - redis.soc_redis
@@ -65,17 +53,31 @@ base:
     - influxdb.adv_influxdb
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
+    - elastalert.soc_elastalert
+    - elastalert.adv_elastalert
     - backup.soc_backup
     - backup.adv_backup
-    - firewall.soc_firewall
+    - curator.soc_curator
-    - firewall.adv_firewall
+    - curator.adv_curator
+    - soctopus.soc_soctopus
+    - soctopus.adv_soctopus
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
   '*_sensor':
     - healthcheck.sensor
-    - soc_global
+    - strelka.soc_strelka
-    - adv_global
+    - strelka.adv_strelka
+    - zeek.soc_zeek
+    - zeek.adv_zeek
+    - bpf.soc_bpf
+    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
+    - suricata.soc_suricata
+    - suricata.adv_suricata
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
@@ -89,15 +91,28 @@ base:
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
     - kibana.secrets
     {% endif %}
-    - soc_global
     - kratos.soc_kratos
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
+    - elastalert.soc_elastalert
+    - elastalert.adv_elastalert
     - manager.soc_manager
     - manager.adv_manager
     - idstools.soc_idstools
     - idstools.adv_idstools
     - soc.soc_soc
+    - soc.adv_soc
+    - soc.license
+    - soctopus.soc_soctopus
+    - soctopus.adv_soctopus
+    - kibana.soc_kibana
+    - kibana.adv_kibana
+    - strelka.soc_strelka
+    - strelka.adv_strelka
+    - curator.soc_curator
+    - curator.adv_curator
     - kratos.soc_kratos
     - kratos.adv_kratos
     - redis.soc_redis
@@ -106,15 +121,19 @@ base:
     - influxdb.adv_influxdb
     - backup.soc_backup
     - backup.adv_backup
-    - firewall.soc_firewall
+    - zeek.soc_zeek
-    - firewall.adv_firewall
+    - zeek.adv_zeek
+    - bpf.soc_bpf
+    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
+    - suricata.soc_suricata
+    - suricata.adv_suricata
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
   '*_standalone':
-    - logstash
+    - logstash.nodes
-    - logstash.manager
-    - logstash.search
     - logstash.soc_logstash
     - logstash.adv_logstash
     - elasticsearch.index_templates
@@ -126,7 +145,6 @@ base:
     {% endif %}
     - secrets
     - healthcheck.standalone
-    - soc_global
     - idstools.soc_idstools
     - idstools.adv_idstools
     - kratos.soc_kratos
@@ -137,51 +155,82 @@ base:
     - influxdb.adv_influxdb
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
+    - elastalert.soc_elastalert
+    - elastalert.adv_elastalert
     - manager.soc_manager
     - manager.adv_manager
     - soc.soc_soc
+    - soc.adv_soc
+    - soc.license
+    - soctopus.soc_soctopus
+    - soctopus.adv_soctopus
+    - kibana.soc_kibana
+    - kibana.adv_kibana
+    - strelka.soc_strelka
+    - strelka.adv_strelka
+    - curator.soc_curator
+    - curator.adv_curator
     - backup.soc_backup
     - backup.adv_backup
-    - firewall.soc_firewall
+    - zeek.soc_zeek
-    - firewall.adv_firewall
+    - zeek.adv_zeek
+    - bpf.soc_bpf
+    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
+    - suricata.soc_suricata
+    - suricata.adv_suricata
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
   '*_heavynode':
     - elasticsearch.auth
-    - soc_global
+    - logstash.nodes
+    - logstash.soc_logstash
+    - logstash.adv_logstash
+    - elasticsearch.soc_elasticsearch
+    - elasticsearch.adv_elasticsearch
+    - curator.soc_curator
+    - curator.adv_curator
     - redis.soc_redis
+    - redis.adv_redis
+    - zeek.soc_zeek
+    - zeek.adv_zeek
+    - bpf.soc_bpf
+    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
+    - suricata.soc_suricata
+    - suricata.adv_suricata
+    - strelka.soc_strelka
+    - strelka.adv_strelka
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
   '*_idh':
-    - soc_global
-    - adv_global
     - idh.soc_idh
     - idh.adv_idh
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
   '*_searchnode':
-    - logstash
+    - logstash.nodes
-    - logstash.search
     - logstash.soc_logstash
     - logstash.adv_logstash
-    - elasticsearch.index_templates
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
     {% endif %}
     - redis.soc_redis
-    - soc_global
+    - redis.adv_redis
-    - adv_global
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
   '*_receiver':
-    - logstash
+    - logstash.nodes
-    - logstash.receiver
     - logstash.soc_logstash
     - logstash.adv_logstash
     {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
@@ -189,8 +238,6 @@ base:
     {% endif %}
     - redis.soc_redis
     - redis.adv_redis
-    - soc_global
-    - adv_global
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
@@ -206,11 +253,21 @@ base:
     - kratos.soc_kratos
     - elasticsearch.soc_elasticsearch
     - elasticsearch.adv_elasticsearch
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
+    - elastalert.soc_elastalert
+    - elastalert.adv_elastalert
     - manager.soc_manager
     - manager.adv_manager
     - soc.soc_soc
-    - soc_global
+    - soc.adv_soc
-    - adv_global
+    - soc.license
+    - soctopus.soc_soctopus
+    - soctopus.adv_soctopus
+    - kibana.soc_kibana
+    - kibana.adv_kibana
+    - curator.soc_curator
+    - curator.adv_curator
     - backup.soc_backup
     - backup.adv_backup
     - kratos.soc_kratos
@@ -219,11 +276,30 @@ base:
     - redis.adv_redis
     - influxdb.soc_influxdb
     - influxdb.adv_influxdb
-    - firewall.soc_firewall
+    - zeek.soc_zeek
-    - firewall.adv_firewall
+    - zeek.adv_zeek
+    - bpf.soc_bpf
+    - bpf.adv_bpf
+    - pcap.soc_pcap
+    - pcap.adv_pcap
+    - suricata.soc_suricata
+    - suricata.adv_suricata
+    - strelka.soc_strelka
+    - strelka.adv_strelka
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}
 
-  '*_workstation':
+  '*_fleet':
+    - backup.soc_backup
+    - backup.adv_backup
+    - logstash.nodes
+    - logstash.soc_logstash
+    - logstash.adv_logstash
+    - elasticfleet.soc_elasticfleet
+    - elasticfleet.adv_elasticfleet
+    - minions.{{ grains.id }}
+    - minions.adv_{{ grains.id }}
+
+  '*_desktop':
     - minions.{{ grains.id }}
     - minions.adv_{{ grains.id }}

salt/_modules/needs_restarting.py

@@ -3,14 +3,14 @@ import subprocess
 
 def check():
 
-  os = __grains__['os']
+  osfam = __grains__['os_family']
   retval = 'False'
 
-  if os == 'Ubuntu':
+  if osfam == 'Debian':
     if path.exists('/var/run/reboot-required'):
       retval = 'True'
 
-  elif os == 'Rocky':
+  elif osfam == 'RedHat':
     cmd = 'needs-restarting -r > /dev/null 2>&1'
     
     try:

salt/allowed_states.map.jinja

@@ -3,16 +3,6 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
-{% set ZEEKVER = salt['pillar.get']('global:mdengine', '') %}
-{% set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
-{% set ELASTALERT = salt['pillar.get']('elastalert:enabled', True) %}
-{% set ELASTICSEARCH = salt['pillar.get']('elasticsearch:enabled', True) %}
-{% set KIBANA = salt['pillar.get']('kibana:enabled', True) %}
-{% set LOGSTASH = salt['pillar.get']('logstash:enabled', True) %}
-{% set CURATOR = salt['pillar.get']('curator:enabled', True) %}
-{% set REDIS = salt['pillar.get']('redis:enabled', True) %}
-{% set STRELKA = salt['pillar.get']('strelka:enabled', '0') %}
 {% set ISAIRGAP = salt['pillar.get']('global:airgap', False) %}
 {% import_yaml 'salt/minion.defaults.yaml' as saltversion %}
 {% set saltversion = saltversion.salt.minion.version %}
@@ -35,6 +25,7 @@
           'soc',
           'kratos',
           'elasticfleet',
+          'elastic-fleet-package-registry',
           'firewall',
           'idstools',
           'suricata.manager',
@@ -55,23 +46,7 @@
           'pcap',
           'suricata',
           'healthcheck',
-          'schedule',
+          'elasticagent',
-          'tcpreplay',
-          'docker_clean'
-          ],
-      'so-helixsensor': [
-          'salt.master',
-          'ca',
-          'ssl',
-          'registry',
-          'telegraf',
-          'firewall',
-          'idstools',
-          'suricata.manager',
-          'zeek',
-          'redis',
-          'elasticsearch',
-          'logstash',
           'schedule',
           'tcpreplay',
           'docker_clean'
@@ -105,7 +80,8 @@
           'schedule',
           'tcpreplay',
           'docker_clean',
-          'elasticfleet'
+          'elasticfleet',
+          'elastic-fleet-package-registry'
           ],
       'so-manager': [
           'salt.master',
@@ -119,6 +95,7 @@
           'soc',
           'kratos',
           'elasticfleet',
+          'elastic-fleet-package-registry',
           'firewall',
           'idstools',
           'suricata.manager',
@@ -137,6 +114,7 @@
           'influxdb',
           'soc',
           'kratos',
+          'elastic-fleet-package-registry',
           'elasticfleet',
           'firewall',
           'manager',
@@ -166,6 +144,7 @@
           'influxdb',
           'soc',
           'kratos',
+          'elastic-fleet-package-registry',
           'elasticfleet',
           'firewall',
           'idstools',
@@ -191,6 +170,16 @@
           'tcpreplay',
           'docker_clean'
           ],
+      'so-fleet': [
+          'ssl',
+          'telegraf',
+          'firewall',
+          'logstash',
+          'healthcheck',
+          'schedule',
+          'elasticfleet',
+          'docker_clean'
+          ],
       'so-receiver': [
           'ssl',
           'telegraf',
@@ -198,31 +187,31 @@
           'schedule',
           'docker_clean'
           ],
-      'so-workstation': [
+      'so-desktop': [
           ],
   }, grain='role') %}
 
-  {% if (PLAYBOOK != 0) and grains.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone'] %}
+  {% if grains.role in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone'] %}
     {% do allowed_states.append('mysql') %}
   {% endif %}
 
-  {%- if ZEEKVER != 'SURICATA' and grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
+  {%- if grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
     {% do allowed_states.append('zeek') %}
   {%- endif %}
 
-  {% if STRELKA and grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
+  {% if grains.role in ['so-sensor', 'so-eval', 'so-standalone', 'so-heavynode'] %}
     {% do allowed_states.append('strelka') %}
   {% endif %}
 
-  {% if ELASTICSEARCH and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import'] %}
+  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-import'] %}
     {% do allowed_states.append('elasticsearch') %}
   {% endif %}
 
-  {% if ELASTICSEARCH and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
+  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
     {% do allowed_states.append('elasticsearch.auth') %}
   {% endif %}
 
-  {% if KIBANA and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
+  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
     {% do allowed_states.append('kibana') %}
     {% do allowed_states.append('kibana.secrets') %}
   {% endif %}
@@ -231,23 +220,19 @@
     {% do allowed_states.append('curator') %}
   {% endif %}
 
-  {% if ELASTALERT and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
+  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
     {% do allowed_states.append('elastalert') %}
   {% endif %}
 
-  {% if (PLAYBOOK !=0) and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
+  {% if grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch'] %}
     {% do allowed_states.append('playbook') %}
   {% endif %}
 
-  {% if (PLAYBOOK !=0) and grains.role in ['so-eval'] %}
+  {% if grains.role in ['so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
-    {% do allowed_states.append('redis') %}
-  {% endif %}
-
-  {% if LOGSTASH and grains.role in ['so-helixsensor', 'so-manager', 'so-standalone', 'so-searchnode', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
     {% do allowed_states.append('logstash') %}
   {% endif %}
 
-  {% if REDIS and grains.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-receiver'] %}
+  {% if grains.role in ['so-manager', 'so-standalone', 'so-managersearch', 'so-heavynode', 'so-receiver', 'so-eval'] %}
     {% do allowed_states.append('redis') %}
   {% endif %}
   

salt/backup/config_backup.sls

@@ -25,6 +25,7 @@ config_backup_script:
 so_config_backup:
   cron.present:
     - name: /usr/sbin/so-config-backup > /dev/null 2>&1
+    - identifier: so_config_backup
     - user: root
     - minute: '1'
     - hour: '0'

salt/ca/init.sls

@@ -18,9 +18,8 @@ include:
 pki_private_key:
   x509.private_key_managed:
     - name: /etc/pki/ca.key
-    - bits: 4096
+    - keysize: 4096
     - passphrase:
-    - cipher: aes_256_cbc
     - backup: True
     {% if salt['file.file_exists']('/etc/pki/ca.key') -%}
     - prereq:
@@ -39,7 +38,7 @@ pki_public_ca_crt:
     - keyUsage: "critical cRLSign, keyCertSign"
     - extendedkeyUsage: "serverAuth, clientAuth"
     - subjectKeyIdentifier: hash
-    - authorityKeyIdentifier: keyid,issuer:always
+    - authorityKeyIdentifier: keyid:always, issuer
     - days_valid: 3650
     - days_remaining: 0
     - backup: True

salt/common/cron/common-rotate

@@ -1,2 +0,0 @@
-#!/bin/bash
-/usr/sbin/logrotate -f /opt/so/conf/log-rotate.conf > /dev/null 2>&1

salt/common/cron/sensor-rotate

@@ -1,2 +0,0 @@
-#!/bin/bash
-/usr/sbin/logrotate -f /opt/so/conf/sensor-rotate.conf > /dev/null 2>&1

salt/common/files/analyst/README

@@ -1,79 +0,0 @@
-The following GUI tools are available on the analyst workstation:
-
-chromium
-  url: https://www.chromium.org/Home
-  To run chromium, click Applications > Internet > Chromium Web Browser
-  
-Wireshark
-  url: https://www.wireshark.org/
-  To run Wireshark, click Applications > Internet > Wireshark Network Analyzer
-
-NetworkMiner
-  url: https://www.netresec.com
-  To run NetworkMiner, click Applications > Internet > NetworkMiner
-
-The following CLI tools are available on the analyst workstation:
-
-bit-twist
-  url: http://bittwist.sourceforge.net
-  To run bit-twist, open a terminal and type: bittwist -h
-
-chaosreader
-  url: http://chaosreader.sourceforge.net
-  To run chaosreader, open a terminal and type: chaosreader -h
-
-dnsiff
-  url: https://www.monkey.org/~dugsong/dsniff/
-  To run dsniff, open a terminal and type: dsniff -h
-
-foremost
-  url: http://foremost.sourceforge.net
-  To run foremost, open a terminal and type: foremost -h
-  
-hping3
-  url: http://www.hping.org/hping3.html
-  To run hping3, open a terminal and type: hping3 -h
-
-netsed
-  url: http://silicone.homelinux.org/projects/netsed/
-  To run netsed, open a terminal and type: netsed -h
-
-ngrep
-  url: https://github.com/jpr5/ngrep
-  To run ngrep, open a terminal and type: ngrep -h
-
-scapy
-  url: http://www.secdev.org/projects/scapy/
-  To run scapy, open a terminal and type: scapy
-
-ssldump
-  url: http://www.rtfm.com/ssldump/
-  To run ssldump, open a terminal and type: ssldump -h
-
-sslsplit
-  url: https://github.com/droe/sslsplit
-  To run sslsplit, open a terminal and type: sslsplit -h
-
-tcpdump
-  url: http://www.tcpdump.org
-  To run tcpdump, open a terminal and type: tcpdump -h
-
-tcpflow
-  url: https://github.com/simsong/tcpflow
-  To run tcpflow, open a terminal and type: tcpflow -h
-
-tcpstat
-  url: https://frenchfries.net/paul/tcpstat/
-  To run tcpstat, open a terminal and type: tcpstat -h
-
-tcptrace
-  url: http://www.tcptrace.org
-  To run tcptrace, open a terminal and type: tcptrace -h
-
-tcpxtract
-  url: http://tcpxtract.sourceforge.net/
-  To run tcpxtract, open a terminal and type: tcpxtract -h
-
-whois
-  url: http://www.linux.it/~md/software/
-  To run whois, open a terminal and type: whois -h

salt/common/files/daemon.json

@@ -1,13 +1,11 @@
-{%- set DOCKERRANGE = salt['pillar.get']('docker:range', '172.17.0.0/24') %}
-{%- set DOCKERBIND = salt['pillar.get']('docker:bip', '172.17.0.1/24') %}
 {
   "registry-mirrors": [
     "https://:5000"
   ],
-  "bip": "{{ DOCKERBIND }}",
+  "bip": "172.17.0.1/24",
   "default-address-pools": [
     {
-      "base": "{{ DOCKERRANGE }}",
+      "base": "172.17.0.0/24",
       "size": 24
     }
   ]

salt/common/files/log-rotate.conf

@@ -1,37 +0,0 @@
-{%- set logrotate_conf = salt['pillar.get']('logrotate:conf') %}
-{%- set group_conf = salt['pillar.get']('logrotate:group_conf') %}
-
-
-/opt/so/log/aptcacher-ng/*.log
-/opt/so/log/idstools/*.log
-/opt/so/log/nginx/*.log
-/opt/so/log/soc/*.log
-/opt/so/log/kratos/*.log
-/opt/so/log/kibana/*.log
-/opt/so/log/influxdb/*.log
-/opt/so/log/elastalert/*.log
-/opt/so/log/soctopus/*.log
-/opt/so/log/curator/*.log
-/opt/so/log/fleet/*.log
-/opt/so/log/suricata/*.log
-/opt/so/log/mysql/*.log
-/opt/so/log/telegraf/*.log
-/opt/so/log/redis/*.log
-/opt/so/log/sensoroni/*.log
-/opt/so/log/stenographer/*.log
-/opt/so/log/salt/so-salt-minion-check
-/opt/so/log/salt/minion
-/opt/so/log/salt/master
-/opt/so/log/logscan/*.log
-/nsm/idh/*.log
-{
-    {{ logrotate_conf | indent(width=4) }}
-}
-
-# Playbook's log directory needs additional configuration
-# because Playbook requires a more permissive directory
-/opt/so/log/playbook/*.log
-{
-    {{ logrotate_conf | indent(width=4) }}
-    {{ group_conf | indent(width=4) }}
-}

salt/common/files/sensor-rotate.conf

@@ -1,22 +0,0 @@
-/opt/so/log/sensor_clean.log
-{
-    daily
-    rotate 2
-    missingok
-    nocompress
-    create
-    sharedscripts
-}
-
-/nsm/strelka/log/strelka.log
-{
-    daily
-    rotate 14
-    missingok
-    copytruncate
-    compress
-    create
-    extension .log
-    dateext
-    dateyesterday
-}

salt/common/init.sls

@@ -10,6 +10,10 @@ include:
   - manager.elasticsearch # needed for elastic_curl_config state
 {% endif %}
 
+net.core.wmem_default:
+  sysctl.present:
+    - value: 26214400
+
 # Remove variables.txt from /tmp - This is temp
 rmvariablesfile:
   file.absent:
@@ -49,13 +53,12 @@ so-status.conf:
     - name: /opt/so/conf/so-status/so-status.conf
     - unless: ls /opt/so/conf/so-status/so-status.conf
 
-sosaltstackperms:
+socore_opso_perms:
   file.directory:
-    - name: /opt/so/saltstack
+    - name: /opt/so
     - user: 939
     - group: 939
-    - dir_mode: 770
+    
-
 so_log_perms:
   file.directory:
     - name: /opt/so/log
@@ -97,6 +100,8 @@ alwaysupdated:
 Etc/UTC:
   timezone.system
 
+# Sync curl configuration for Elasticsearch authentication
+{% if GLOBALS.role in ['so-eval', 'so-heavynode', 'so-import', 'so-manager', 'so-managersearch', 'so-searchnode', 'so-standalone'] %}
 elastic_curl_config:
   file.managed:
     - name: /opt/so/conf/elasticsearch/curl.config
@@ -108,22 +113,25 @@ elastic_curl_config:
     - require:
       - file: elastic_curl_config_distributed
   {% endif %}
+{% endif %}
 
-# Sync some Utilities
+
-utilsyncscripts:
+common_sbin:
   file.recurse:
     - name: /usr/sbin
-    - user: root
+    - source: salt://common/tools/sbin
-    - group: root
+    - user: 939
+    - group: 939
+    - file_mode: 755
+
+common_sbin_jinja:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://common/tools/sbin_jinja
+    - user: 939
+    - group: 939 
     - file_mode: 755
     - template: jinja
-    - source: salt://common/tools/sbin
-    - exclude_pat:
-        - so-common
-        - so-firewall
-        - so-image-common
-        - soup
-        - so-status
 
 so-status_script:
   file.managed:
@@ -133,60 +141,18 @@ so-status_script:
 
 {% if GLOBALS.role in GLOBALS.sensor_roles %}
 # Add sensor cleanup
-/usr/sbin/so-sensor-clean:
+so-sensor-clean:
   cron.present:
+    - name: /usr/sbin/so-sensor-clean
+    - identifier: so-sensor-clean
     - user: root
     - minute: '*'
     - hour: '*'
     - daymonth: '*'
     - month: '*'
     - dayweek: '*'
-
-sensorrotatescript:
-  file.managed:
-    - name: /usr/local/bin/sensor-rotate
-    - source: salt://common/cron/sensor-rotate
-    - mode: 755
-
-sensorrotateconf:
-  file.managed:
-    - name: /opt/so/conf/sensor-rotate.conf
-    - source: salt://common/files/sensor-rotate.conf
-    - mode: 644
-
-/usr/local/bin/sensor-rotate:
-  cron.present:
-    - user: root
-    - minute: '1'
-    - hour: '0'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-
 {% endif %}
 
-commonlogrotatescript:
-  file.managed:
-    - name: /usr/local/bin/common-rotate
-    - source: salt://common/cron/common-rotate
-    - mode: 755
-
-commonlogrotateconf:
-  file.managed:
-    - name: /opt/so/conf/log-rotate.conf
-    - source: salt://common/files/log-rotate.conf
-    - template: jinja
-    - mode: 644
-
-/usr/local/bin/common-rotate:
-  cron.present:
-    - user: root
-    - minute: '1'
-    - hour: '0'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-
 # Create the status directory
 sostatusdir:
   file.directory:
@@ -200,17 +166,11 @@ sostatus_log:
     - name: /opt/so/log/sostatus/status.log
     - mode: 644
 
-common_pip_dependencies:
+# Install sostatus check cron. This is used to populate Grid.
-  pip.installed:
+so-status_check_cron:
-    - user: root
-    - pkgs: 
-      - rich
-    - target: /usr/lib64/python3.6/site-packages
-
-# Install sostatus check cron
-sostatus_check_cron:
   cron.present:
     - name: '/usr/sbin/so-status -j > /opt/so/log/sostatus/status.log 2>&1'
+    - identifier: so-status_check_cron
     - user: root
     - minute: '*/1'
     - hour: '*'
@@ -220,7 +180,7 @@ sostatus_check_cron:
 
 remove_post_setup_cron:
   cron.absent:
-    - name: 'salt-call state.highstate'
+    - name: 'PATH=$PATH:/usr/sbin salt-call state.highstate'
     - identifier: post_setup_cron
 
 {% if GLOBALS.role not in ['eval', 'manager', 'managersearch', 'standalone'] %}
@@ -234,8 +194,8 @@ soversionfile:
     
 {% endif %}
 
-{% if GLOBALS.so_model %}
+{% if GLOBALS.so_model and GLOBALS.so_model not in ['SO2AMI01', 'SO2AZI01', 'SO2GCI01'] %}
-  {% if GLOBALS.os == 'Rocky' %}     
+  {% if GLOBALS.os == 'OEL' %}     
 # Install Raid tools
 raidpkgs:
   pkg.installed:
@@ -246,9 +206,10 @@ raidpkgs:
   {% endif %}
 
 # Install raid check cron
-so_raid_status:
+so-raid-status:
   cron.present:
     - name: '/usr/sbin/so-raid-status > /dev/null 2>&1'
+    - identifier: so-raid-status
     - user: root
     - minute: '*/15'
     - hour: '*'
@@ -256,8 +217,7 @@ so_raid_status:
     - month: '*'
     - dayweek: '*'
 
-{% endif %}
+  {% endif %}
-
 {% else %}
 
 {{sls}}_state_not_allowed:

salt/common/packages.sls

@@ -1,56 +1,88 @@
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 
-{% if GLOBALS.os == 'Ubuntu' %}
+{% if GLOBALS.os_family == 'Debian' %}
 commonpkgs:
   pkg.installed:
     - skip_suggestions: True
     - pkgs:
-      - chrony
       - apache2-utils
       - wget
       - ntpdate
       - jq
-      - python3-docker
       - curl
       - ca-certificates
       - software-properties-common
       - apt-transport-https
       - openssl
-      - netcat
+      - netcat-openbsd
-      - python3-mysqldb
       - sqlite3
       - libssl-dev
+      - procps
       - python3-dateutil
-      - python3-m2crypto
+      - python3-docker
-      - python3-mysqldb
       - python3-packaging
+      - python3-watchdog
       - python3-lxml
       - git
+      - rsync
       - vim
-{% elif GLOBALS.os == 'Rocky' %}     
+      - tar
+      - unzip
+      {% if grains.oscodename != 'focal' %}
+      - python3-rich
+      {% endif %}
+
+{%     if grains.oscodename == 'focal' %}
+# since Ubuntu requires and internet connection we can use pip to install modules
+python3-pip:
+  pkg.installed
+
+python-rich:
+  pip.installed:
+    - name: rich
+    - target: /usr/local/lib/python3.8/dist-packages/
+    - require:
+      - pkg: python3-pip
+{%     endif %}
+{% endif %}
+
+{% if GLOBALS.os_family == 'RedHat' %}
 commonpkgs:
   pkg.installed:
     - skip_suggestions: True
     - pkgs:
-      - wget
-      - jq
-      - tcpdump
-      - httpd-tools
-      - net-tools
       - curl
-      - sqlite
-      - mariadb-devel
-      - python3-dnf-plugin-versionlock
-      - nmap-ncat
-      - yum-utils
       - device-mapper-persistent-data
-      - lvm2
+      - fuse
-      - openssl
+      - fuse-libs
+      - fuse-overlayfs
+      - fuse-common
+      - fuse3
+      - fuse3-libs
       - git
+      - httpd-tools
+      - jq
+      - lvm2
+      {% if GLOBALS.os == 'CentOS Stream' %}
+      - MariaDB-devel
+      {% else %}
+      - mariadb-devel
+      {% endif %}
+      - net-tools
+      - nmap-ncat
+      - openssl
+      - procps-ng
+      - python3-dnf-plugin-versionlock
       - python3-docker
       - python3-m2crypto
-      - rsync
+      - python3-packaging
+      - python3-pyyaml
       - python3-rich
       - python3-watchdog
+      - rsync
+      - sqlite
+      - tcpdump
       - unzip
+      - wget
+      - yum-utils
 {% endif %}

salt/common/soup_scripts.sls

@@ -8,6 +8,15 @@ soup_scripts:
     - source: salt://common/tools/sbin
     - include_pat:
         - so-common
-        - so-firewall
         - so-image-common
-        - soup
+
+soup_manager_scripts:
+  file.recurse:
+    - name: /usr/sbin
+    - user: root
+    - group: root
+    - file_mode: 755
+    - source: salt://manager/tools/sbin
+    - include_pat:
+        - so-firewall
+        - soup

salt/common/tools/sbin/so-common

@@ -5,6 +5,16 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
+# Elastic agent is not managed by salt. Because of this we must store this base information in a 
+# script that accompanies the soup system. Since so-common is one of those special soup files, 
+# and since this same logic is required during installation, it's included in this file.
+ELASTIC_AGENT_TARBALL_VERSION="8.8.2"
+ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz"
+ELASTIC_AGENT_MD5="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5"
+ELASTIC_AGENT_EXPANSION_DIR=/nsm/elastic-fleet/artifacts/beats/elastic-agent
+
 DEFAULT_SALT_DIR=/opt/so/saltstack/default
 DOC_BASE_URL="https://docs.securityonion.net/en/2.4"
 
@@ -54,33 +64,37 @@ add_interface_bond0() {
 			ethtool -K "$BNIC" $i off &>/dev/null
 		fi
 	done
-	# Check if the bond slave connection has already been created
-	nmcli -f name,uuid -p con | grep -q "bond0-slave-$BNIC"
-	local found_int=$?
 
-	if [[ $found_int != 0 ]]; then
+        if ! [[ $is_cloud ]]; then
-		# Create the slave interface and assign it to the bond
+	  # Check if the bond slave connection has already been created
-		nmcli con add type ethernet ifname "$BNIC" con-name "bond0-slave-$BNIC" master bond0 -- \
+	  nmcli -f name,uuid -p con | grep -q "bond0-slave-$BNIC"
-			ethernet.mtu "$MTU" \
+	  local found_int=$?
-			connection.autoconnect "yes"
-	else
-		local int_uuid
-		int_uuid=$(nmcli -f name,uuid -p con | sed -n "s/bond0-slave-$BNIC //p" | tr -d ' ')
 
-		nmcli con mod "$int_uuid" \
+	  if [[ $found_int != 0 ]]; then
-			ethernet.mtu "$MTU" \
+		  # Create the slave interface and assign it to the bond
-			connection.autoconnect "yes"
+		  nmcli con add type ethernet ifname "$BNIC" con-name "bond0-slave-$BNIC" master bond0 -- \
-	fi
+			  ethernet.mtu "$MTU" \
+			  connection.autoconnect "yes"
+	  else
+		  local int_uuid
+		  int_uuid=$(nmcli -f name,uuid -p con | sed -n "s/bond0-slave-$BNIC //p" | tr -d ' ')
+
+		  nmcli con mod "$int_uuid" \
+			  ethernet.mtu "$MTU" \
+			  connection.autoconnect "yes"
+	  fi
+        fi
 
 	ip link set dev "$BNIC" arp off multicast off allmulticast off promisc on
-			
+
-	# Bring the slave interface up
+        if ! [[ $is_cloud ]]; then
-	if [[ $verbose == true ]]; then
+	  # Bring the slave interface up
-		nmcli con up "bond0-slave-$BNIC"
+	  if [[ $verbose == true ]]; then
-	else
+		  nmcli con up "bond0-slave-$BNIC"
-		nmcli con up "bond0-slave-$BNIC" &>/dev/null
+	  else
+		  nmcli con up "bond0-slave-$BNIC" &>/dev/null
+	  fi
 	fi
-	 
 	if [ "$nic_error" != 0 ]; then
 		return "$nic_error"
 	fi
@@ -156,6 +170,34 @@ disable_fastestmirror() {
 	sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
 }
 
+download_and_verify() {
+	source_url=$1
+	source_md5_url=$2
+	dest_file=$3
+	md5_file=$4
+	expand_dir=$5
+
+	if [[ -n "$expand_dir" ]]; then
+		mkdir -p "$expand_dir"
+	fi
+
+	if ! verify_md5_checksum "$dest_file" "$md5_file"; then
+		retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L '$source_url' --output '$dest_file'" "" ""
+		retry 15 10 "curl --fail --retry 5 --retry-delay 15 -L '$source_md5_url' --output '$md5_file'"  "" ""
+
+		if verify_md5_checksum "$dest_file" "$md5_file"; then
+		  echo "Source file and checksum are good."
+		else
+		  echo "Unable to download and verify the source file and checksum."
+		  return 1
+		fi
+	fi
+
+	if [[ -n "$expand_dir" ]]; then
+		tar -xf "$dest_file" -C "$expand_dir"
+	fi
+}
+
 elastic_license() {
 
 read -r -d '' message <<- EOM
@@ -194,19 +236,20 @@ get_random_value() {
 }
 
 gpg_rpm_import() {
-	if [[ "$OS" == "rocky" ]]; then
+	if [[ $is_oracle ]]; then
 	    if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then
-	        local RPMKEYSLOC="../salt/repo/client/files/rocky/keys"
+	        local RPMKEYSLOC="../salt/repo/client/files/$OS/keys"
 	    else
-	        local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/rocky/keys"
+	        local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/$OS/keys"
 	    fi
-	
+		RPMKEYS=('RPM-GPG-KEY-oracle' 'RPM-GPG-KEY-EPEL-9' 'SALT-PROJECT-GPG-PUBKEY-2023.pub' 'docker.pub' 'securityonion.pub' 'MariaDB-Server-GPG-KEY')
-	    RPMKEYS=('RPM-GPG-KEY-EPEL-9' 'SALTSTACK-GPG-KEY2.pub' 'docker.pub' 'securityonion.pub')
+		for RPMKEY in "${RPMKEYS[@]}"; do
-
-	    for RPMKEY in "${RPMKEYS[@]}"; do
     	        rpm --import $RPMKEYSLOC/$RPMKEY
 	        echo "Imported $RPMKEY"
 	    done
+	elif [[ $is_rpm ]]; then
+		echo "Importing the security onion GPG key"
+		rpm --import ../salt/repo/client/files/oracle/keys/securityonion.pub
 	fi
 }
 
@@ -219,12 +262,15 @@ init_monitor() {
 	
 	if [[ $MONITORNIC == "bond0" ]]; then 
 	  BIFACES=$(lookup_bond_interfaces)
+	  for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload generic-receive-offload generic-segmentation-offload tcp-segmentation-offload; do
+	    ethtool -K "$MONITORNIC" "$i" off;
+  	  done
 	else
 	  BIFACES=$MONITORNIC
 	fi
 
 	for DEVICE_IFACE in $BIFACES; do
-	  for i in rx tx sg tso ufo gso gro lro; do
+	  for i in rx tx sg tso ufo gso gro lro rx-vlan-offload tx-vlan-offload generic-receive-offload generic-segmentation-offload tcp-segmentation-offload; do
 	    ethtool -K "$DEVICE_IFACE" "$i" off;
   	  done
 	  ip link set dev "$DEVICE_IFACE" arp off multicast off allmulticast off promisc on
@@ -238,7 +284,7 @@ is_manager_node() {
 is_sensor_node() {
 	# Check to see if this is a sensor (forward) node
 	is_single_node_grid && return 0
-	grep "role: so-" /etc/salt/grains | grep -E "sensor|heavynode|helix" &> /dev/null
+	grep "role: so-" /etc/salt/grains | grep -E "sensor|heavynode" &> /dev/null
 }
 
 is_single_node_grid() {
@@ -296,6 +342,17 @@ lookup_role() {
 	echo ${pieces[1]}
 }
 
+is_feature_enabled() {
+	feature=$1
+	enabled=$(lookup_salt_value features)
+	for cur in $enabled; do
+		if [[ "$feature" == "$cur" ]]; then
+			return 0
+		fi
+	done
+	return 1
+}
+
 require_manager() {
 	if is_manager_node; then
 		echo "This is a manager, so we can proceed."
@@ -379,19 +436,22 @@ salt_minion_count() {
 
 }
 
-set_cron_service_name() {
-	if [[ "$OS" == "rocky" ]]; then
-		cron_service_name="crond"
-	else
-		cron_service_name="cron"
-	fi
-}
-
 set_os() {
 	if [ -f /etc/redhat-release ]; then
-		OS=rocky
+		if grep -q "Rocky Linux release 9" /etc/redhat-release; then
+			OS=rocky
+			OSVER=9
+			is_rocky=true
+		elif grep -q "CentOS Stream release 9" /etc/redhat-release; then
+			OS=centos
+			OSVER=9
+			is_centos=true
+		fi
+		cron_service_name="crond"
 	else
 		OS=ubuntu
+		is_ubuntu=true
+		cron_service_name="cron"
 	fi
 }
 
@@ -400,7 +460,7 @@ set_minionid() {
 }
 
 set_palette() {
-	if [ "$OS" == ubuntu ]; then
+	if [[ $is_deb ]]; then
 	    update-alternatives --set newt-palette /etc/newt/palette.original
     fi
 }
@@ -447,6 +507,11 @@ has_uppercase() {
 		|| return 1
 }
 
+update_elastic_agent() {
+	echo "Checking if Elastic Agent update is necessary..."
+	download_and_verify "$ELASTIC_AGENT_URL" "$ELASTIC_AGENT_MD5_URL" "$ELASTIC_AGENT_FILE" "$ELASTIC_AGENT_MD5" "$ELASTIC_AGENT_EXPANSION_DIR"
+}
+
 valid_cidr() {
 	# Verify there is a backslash in the string
 	echo "$1" | grep -qP "^[^/]+/[^/]+$" || return 1
@@ -600,6 +665,23 @@ valid_username() {
 	echo "$user" | grep -qP '^[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}\$)$' && return 0 || return 1
 }
 
+verify_md5_checksum() {
+	data_file=$1
+	md5_file=${2:-${data_file}.md5}
+
+	if [[ ! -f "$dest_file" || ! -f "$md5_file" ]]; then
+		return 2
+	fi
+
+	SOURCEHASH=$(md5sum "$data_file" | awk '{ print $1 }')
+	HASH=$(cat "$md5_file")
+
+	if [[ "$HASH" == "$SOURCEHASH" ]]; then
+		return 0
+	fi
+	return 1
+}
+
 wait_for_web_response() {
 	url=$1
 	expected=$2

salt/common/tools/sbin/so-elastic-agent-gen-installers

@@ -1,34 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
-# this file except in compliance with the Elastic License 2.0.
-
-#so-elastic-agent-gen-installers $FleetHost $EnrollmentToken 
-
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
-. /usr/sbin/so-common
-
-ENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints")) | .api_key')
-
-FLEETHOST=$(lookup_pillar "server:url" "elasticfleet")
-
-#FLEETHOST=$1
-#ENROLLMENTOKEN=$2
-CONTAINERGOOS=( "linux" "darwin" "windows" )
-
-rm -rf /tmp/elastic-agent-workspace
-mkdir -p /tmp/elastic-agent-workspace
-
-for OS in "${CONTAINERGOOS[@]}"
-do
-    printf "\n\nGenerating $OS Installer..."
-    cp /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/so-elastic-agent-*-$OS-x86_64.tar.gz /tmp/elastic-agent-workspace/$OS.tar.gz
-    docker run -e CGO_ENABLED=0 -e GOOS=$OS \
-    --mount type=bind,source=/etc/ssl/certs/,target=/workspace/files/cert/ \
-    --mount type=bind,source=/tmp/elastic-agent-workspace/,target=/workspace/files/elastic-agent/ \
-    --mount type=bind,source=/opt/so/saltstack/local/salt/elasticfleet/files/so_agent-installers/,target=/output/ \
-    {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-elastic-agent-builder:{{ GLOBALS.so_version }} go build -ldflags "-X main.fleetHost=$FLEETHOST -X main.enrollmentToken=$ENROLLMENTOKEN"  -o /output/so-elastic-agent_$OS
-    printf "\n $OS Installer Generated..."
-done

salt/common/tools/sbin/so-elastic-clear

@@ -1,94 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
-. /usr/sbin/so-common
-
-SKIP=0
-#########################################
-# Options
-#########################################
-usage()
-{
-cat <<EOF
-Security Onion Elastic Clear
-  Options:
-  -h         This message
-  -y         Skip interactive mode
-EOF
-}
-while getopts "h:y" OPTION
-do
-        case $OPTION in
-                h)
-                        usage
-                        exit 0
-                        ;;
-                
-                y)
-                        SKIP=1
-                        ;;
-                *)
-                        usage
-                        exit 0
-                        ;;
-        esac
-done
-if [ $SKIP -ne 1 ]; then
-        # List indices
-        echo 
-        curl -K /opt/so/conf/elasticsearch/curl.config -k -L https://{{ NODEIP }}:9200/_cat/indices?v
-        echo
-        # Inform user we are about to delete all data
-        echo
-        echo "This script will delete all data (documents, indices, etc.) in the Elasticsearch database."
-        echo
-        echo "If you would like to proceed, please type "AGREE" and hit ENTER."
-        echo
-        # Read user input
-        read INPUT
-        if [ "$INPUT" != "AGREE" ] ; then exit 0; fi
-fi
-
-# Check to see if Logstash are running
-LS_ENABLED=$(so-status | grep logstash)
-EA_ENABLED=$(so-status | grep elastalert)
-
-if [ ! -z "$LS_ENABLED" ]; then
-
-        /usr/sbin/so-logstash-stop
-
-fi
-
-if [ ! -z "$EA_ENABLED" ]; then
-
-        /usr/sbin/so-elastalert-stop
-
-fi
-
-# Delete data
-echo "Deleting data..."
-
-INDXS=$(curl -K /opt/so/conf/elasticsearch/curl.config -s -XGET -k -L https://{{ NODEIP }}:9200/_cat/indices?v | egrep 'logstash|elastalert|so-' | awk '{ print $3 }')
-for INDX in ${INDXS}
-do
-    curl -K /opt/so/conf/elasticsearch/curl.config-XDELETE -k -L https://"{{ NODEIP }}:9200/${INDX}" > /dev/null 2>&1
-done
-
-#Start Logstash
-if [ ! -z "$LS_ENABLED" ]; then
-
-        /usr/sbin/so-logstash-start
-
-fi
-
-if [ ! -z "$EA_ENABLED" ]; then
-
-        /usr/sbin/so-elastalert-start
-
-fi
-

salt/common/tools/sbin/so-elastic-fleet-integration-policy-load

@@ -1,137 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-{%- set ZEEKVER = salt['pillar.get']('global:mdengine', '') %}
-{%- set STRELKAENABLED = salt['pillar.get']('strelka:enabled', '0') %}
-{%- set RITAENABLED = salt['pillar.get']('rita:enabled', False) %}
-
-wait_for_web_response "http://localhost:5601/api/spaces/space/default" "default" 300 "curl -K /opt/so/conf/elasticsearch/curl.config"
-
-# Let's snag a cookie from Kibana
-SESSIONCOOKIE=$(curl -K /opt/so/conf/elasticsearch/curl.config -c - -X GET http://localhost:5601/ | grep sid | awk '{print $7}')
-
-# Disable certain Features from showing up in the Kibana UI
-echo
-echo "Disable certain Features from showing up in the Kibana UI"
-so-kibana-space-defaults
-echo
-
-# Suricata logs
-echo
-echo "Setting up Suricata package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "suricata-logs", "name": "suricata-logs", "description": "Suricata integration", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/suricata/eve*.json" ], "data_stream.dataset": "suricata", "tags": [],"processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata", "custom": "pipeline: suricata.common" }}}}}}'
-echo
-
-# Zeek logs
-echo
-echo "Setting up Zeek package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "zeek-logs", "description": "Zeek logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/zeek/logs/current/*.log"], "data_stream.dataset": "zeek", "tags": [], "processors": "- dissect:\n    tokenizer: \"/nsm/zeek/logs/current/%{pipeline}.log\"\n    field: \"log.file.path\"\n    trim_chars: \".log\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"pipeline\");\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        pipeline: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n" } } } } } }'
-echo
-
-
-# Import - EVTX
-echo
-echo "Setting up EVTX import package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "package": { "name": "log", "version": "1.1.0" }, "name": "import-evtx-logs", "namespace": "so", "description": "Import Windows EVTX logs", "policy_id": "so-grid-nodes", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/import/*/evtx/data.json" ], "data_stream.dataset": "import", "custom": "pipeline: import.wel", "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/evtx/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n- add_fields:\n    target: event\n    fields:\n      module: windows_eventlog\n      imported: true", "tags": [] } } } } } }'
-echo
-
-# Import - Suricata logs
-echo
-echo "Setting up Suricata import package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "import-suricata-logs", "name": "import-suricata-logs", "description": "Import Suricata logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/suricata/eve*.json"], "data_stream.dataset": "import", "tags": [], "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: suricata\n      imported: true\n- dissect:\n      tokenizer: \"/nsm/import/%{import.id}/suricata/%{import.file}\"\n      field: \"log.file.path\"\n      target_prefix: \"\"", "custom": "pipeline: suricata.common" } } } } } }'
-echo
-
-# Import - Zeek logs
-echo
-echo "Setting up Zeek import package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "name": "import-zeek-logs", "description": "Zeek Import logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": ["/nsm/import/*/zeek/logs/*.log"], "data_stream.dataset": "import", "tags": [], "processors": "- dissect:\n    tokenizer: \"/nsm/import/%{import.id}/zeek/logs/%{import.file}\"\n    field: \"log.file.path\"\n    target_prefix: \"\"\n- script:\n      lang: javascript\n      source: >\n        function process(event) {\n          var pl = event.Get(\"import.file\").slice(0,-4);\n          event.Put(\"@metadata.pipeline\", \"zeek.\" + pl);\n        }\n- add_fields:\n    target: event\n    fields:\n      category: network\n      module: zeek\n      imported: true\n- add_tags:\n    tags: \"ics\"\n    when:\n      regexp:\n        import.file: \"^bacnet*|^bsap*|^cip*|^cotp*|^dnp3*|^ecat*|^enip*|^modbus*|^opcua*|^profinet*|^s7comm*\"", "custom": "exclude_files: [\"broker|capture_loss|ecat_arp_info|loaded_scripts|packet_filter|stats|stderr|stdout.log$\"]\n" } } } } } }'
-echo
-
-# Strelka logs
-echo
-echo "Setting up Strelka package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "strelka-logs", "name": "strelka-logs", "description": "Strelka logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/strelka/log/strelka.log" ], "data_stream.dataset": "strelka", "tags": [],"processors": "- add_fields:\n    target: event\n    fields:\n      category: file\n      module: strelka", "custom": "pipeline: strelka.file" }}}}}}'
-echo
-
-# Syslog TCP Port 514
-echo
-echo "Setting up Syslog TCP package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "tcp", "version": "1.5.0" }, "id": "syslog-tcp-514", "name": "syslog-tcp-514", "description": "Syslog Over TCP Port 514", "namespace": "so", "inputs": { "tcp-tcp": { "enabled": true, "streams": { "tcp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "processors": "- add_fields:\n    target: event\n    fields:\n      module: syslog", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }'
-echo
-
-# Syslog UDP Port 514
-echo
-echo "Setting up Syslog UDP package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "policy_id": "so-grid-nodes", "package": { "name": "udp", "version": "1.5.0" }, "id": "syslog-udp-514", "name": "syslog-udp-514", "description": "Syslog over UDP Port 514", "namespace": "so", "inputs": { "udp-udp": { "enabled": true, "streams": { "udp.generic": { "enabled": true, "vars": { "listen_address": "0.0.0.0", "listen_port": "514", "data_stream.dataset": "syslog", "pipeline": "syslog", "max_message_size": "10KiB", "keep_null": false, "processors": "- add_fields:\n    target: event\n    fields: \n      module: syslog\n", "tags": [ "syslog" ], "syslog_options": "field: message\n#format: auto\n#timezone: Local" } } } } } }'
-echo
-
-# Kratos logs
-echo
-echo "Setting up Kratos package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "kratos-logs", "name": "kratos-logs", "description": "Kratos logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kratos/kratos.log" ], "data_stream.dataset": "kratos", "tags": [],"custom":"pipeline: kratos","processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: iam\n      module: kratos" }}}}}}'
-echo
-
-# RITA Logs
-#echo
-#echo "Setting up RITA package policy..."
-#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n    target: event\n    fields:\n      category: network\n      module: rita\n- if:\n    log.file.path: beacons.csv\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.beacon\n- if:\n    regexp:\n      log.file.path: \"*connections.csv\"\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.connection\n- if:\n    log.file.path: \"exploded-dns.csv\"\n  then: \n    - add_fields:\n        target: \"@metadata\"\n        fields:\n          pipeline: rita.dns" }}}}}}'
-#echo
-
-# Elasticsearch logs
-echo
-echo "Setting up Elasticsearch package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "elasticsearch", "version": "1.0.0" }, "id": "elasticsearch-logs", "name": "elasticsearch-logs", "description": "Elasticsearch Logs", "namespace": "default", "inputs": { "elasticsearch-logfile": { "enabled": true, "streams": { "elasticsearch.audit": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_audit.json" ] } }, "elasticsearch.deprecation": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_deprecation.json" ] } }, "elasticsearch.gc": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/gc.log.[0-9]*", "/var/log/elasticsearch/gc.log" ] } }, "elasticsearch.server": { "enabled": true, "vars": { "paths": [ "/opt/so/log/elasticsearch/*.log" ] } }, "elasticsearch.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/elasticsearch/*_index_search_slowlog.json", "/var/log/elasticsearch/*_index_indexing_slowlog.json" ] } } } }, "elasticsearch-elasticsearch/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9200" ], "scope": "node" }, "streams": { "elasticsearch.stack_monitoring.ccr": { "enabled": false }, "elasticsearch.stack_monitoring.cluster_stats": { "enabled": false }, "elasticsearch.stack_monitoring.enrich": { "enabled": false }, "elasticsearch.stack_monitoring.index": { "enabled": false }, "elasticsearch.stack_monitoring.index_recovery": { "enabled": false, "vars": { "active.only": true } }, "elasticsearch.stack_monitoring.index_summary": { "enabled": false }, "elasticsearch.stack_monitoring.ml_job": { "enabled": false }, "elasticsearch.stack_monitoring.node": { "enabled": false }, "elasticsearch.stack_monitoring.node_stats": { "enabled": false }, "elasticsearch.stack_monitoring.pending_tasks": { "enabled": false }, "elasticsearch.stack_monitoring.shard": { "enabled": false } } } } }'
-echo
-
-# Logstash logs
-#echo
-#echo "Setting up Logstash package policy..."
-#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "logstash", "version": "2.0.0" }, "id": "logstash-logs", "name": "logstash-logs", "description": "Logstash logs", "namespace": "default", "inputs": { "logstash-logfile": { "enabled": true, "streams": { "logstash.log": { "enabled": true, "vars": { "paths": [ "/opt/so/logs/logstash/logstash.log" ] } }, "logstash.slowlog": { "enabled": false, "vars": { "paths": [ "/var/log/logstash/logstash-slowlog-plain*.log", "/var/log/logstash/logstash-slowlog-json*.log" ] } } } }, "logstash-logstash/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:9600" ], "period": "10s" }, "streams": { "logstash.stack_monitoring.node": { "enabled": false }, "logstash.stack_monitoring.node_stats": { "enabled": false } } } } }'
-#echo
-
-# Kibana logs
-#echo
-#echo "Setting up Kibana package policy..."
-#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "kibana", "version": "2.0.0" }, "id": "kibana-logs", "name": "kibana-logs", "description": "Kibana logs", "namespace": "default", "inputs": { "kibana-logfile": { "enabled": true, "streams": { "kibana.audit": { "enabled": false, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } }, "kibana.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/kibana/kibana.log" ] } } } }, "kibana-kibana/metrics": { "enabled": false, "vars": { "hosts": [ "http://localhost:5601" ] }, "streams": { "kibana.stack_monitoring.cluster_actions": { "enabled": false }, "kibana.stack_monitoring.cluster_rules": { "enabled": false }, "kibana.stack_monitoring.node_actions": { "enabled": false }, "kibana.stack_monitoring.node_rules": { "enabled": false }, "kibana.stack_monitoring.stats": { "enabled": false }, "kibana.stack_monitoring.status": { "enabled": false } } } } }'
-#echo
-
-# Redis logs
-echo
-echo "Setting up Redis package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "redis", "version": "1.4.0" }, "id": "redis-logs", "name": "redis-logs", "description": "Redis logs", "namespace": "default", "inputs": { "redis-logfile": { "enabled": true, "streams": { "redis.log": { "enabled": true, "vars": { "paths": [ "/opt/so/log/redis/redis.log" ], "tags": [ "redis-log" ], "preserve_original_event": false } } } }, "redis-redis": { "enabled": false, "streams": { "redis.slowlog": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "password": "" } } } }, "redis-redis/metrics": { "enabled": false, "vars": { "hosts": [ "127.0.0.1:6379" ], "idle_timeout": "20s", "maxconn": 10, "network": "tcp", "password": "" }, "streams": { "redis.info": { "enabled": false, "vars": { "period": "10s" } }, "redis.key": { "enabled": false, "vars": { "key.patterns": "- limit: 20\n  pattern: '*'\n", "period": "10s" } }, "redis.keyspace": { "enabled": false, "vars": { "period": "10s" } } } } } }'
-echo
-
-# IDH logs
-echo
-echo "Setting up IDH package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"policy_id":"so-grid-nodes","package":{"name":"log","version":"1.1.1"},"id":"idh-logs","name":"idh-logs","namespace":"so","description":"IDH integration","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/nsm/idh/opencanary.log"],"data_stream.dataset":"idh","custom":"pipeline: common","processors": "\n- decode_json_fields:\n    fields: [\"message\"]\n    target: \"\"\n    add_error_key: true\n- drop_fields:\n    when:\n      equals:\n        logtype: \"1001\"\n    fields: [\"src_host\", \"src_port\", \"dst_host\", \"dst_port\" ]\n    ignore_missing: true\n- rename:\n    fields:\n      - from: \"src_host\"\n        to: \"source.ip\"\n      - from: \"src_port\"\n        to: \"source.port\"\n      - from: \"dst_host\"\n        to: \"destination.host\"\n      - from: \"dst_port\"\n        to: \"destination.port\"\n    ignore_missing: true\n- convert:\n    fields:\n      - {from: \"logtype\", to: \"event.code\", type: \"string\"}\n    ignore_missing: true\n- drop_fields:\n    fields: '\''[\"prospector\", \"input\", \"offset\", \"beat\"]'\''\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: opencanary","tags":[]}}}}}}'
-echo
-
-# SOC - Server logs
-echo
-echo "Setting up SOC - Server Logs package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"package":{"name":"log","version":"1.1.2"},"name":"soc-server-logs","namespace":"so","description":"Security Onion Console Logs","policy_id":"so-grid-nodes","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/opt/so/log/soc/sensoroni-server.log"],"data_stream.dataset":"soc","custom":"pipeline: common","processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"soc\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: server\n- rename:\n    fields:\n      - from: \"soc.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"soc.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"soc.fields.method\"\n        to: \"http.request.method\"\n      - from: \"soc.fields.path\"\n        to: \"url.path\"\n      - from: \"soc.message\"\n        to: \"event.action\"\n      - from: \"soc.level\"\n        to: \"log.level\"\n    ignore_missing: true","tags":[]}}}}}}'
-echo
-
-# SOC - Sensoroni logs
-echo
-echo "Setting up SOC - Sensoroni Logs package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"package":{"name":"log","version":"1.1.2"},"name":"soc-sensoroni-logs","namespace":"so","description":"Security Onion - Sensoroni - Logs","policy_id":"so-grid-nodes","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/opt/so/log/sensoroni/sensoroni.log"],"data_stream.dataset":"soc","custom":"pipeline: common","processors": "- decode_json_fields:\n    fields: [\"message\"]\n    target: \"sensoroni\"\n    process_array: true\n    max_depth: 2\n    add_error_key: true      \n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: sensoroni\n- rename:\n    fields:\n      - from: \"sensoroni.fields.sourceIp\"\n        to: \"source.ip\"\n      - from: \"sensoroni.fields.status\"\n        to: \"http.response.status_code\"\n      - from: \"sensoroni.fields.method\"\n        to: \"http.request.method\"\n      - from: \"sensoroni.fields.path\"\n        to: \"url.path\"\n      - from: \"sensoroni.message\"\n        to: \"event.action\"\n      - from: \"sensoroni.level\"\n        to: \"log.level\"\n    ignore_missing: true","tags":[]}}}}}}'
-echo
-
-# SOC - Elastic Auth Sync logs
-echo
-echo "Setting up SOC - Elastic Auth Sync Logs package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"package":{"name":"log","version":"1.1.2"},"name":"soc-auth-sync-logs","namespace":"so","description":"Security Onion - Elastic Auth Sync - Logs","policy_id":"so-grid-nodes","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/opt/so/log/soc/sync.log"],"data_stream.dataset":"soc","custom":"pipeline: common","processors": "- dissect:\n    tokenizer: \"%{event.action}\"\n    field: \"message\"\n    target_prefix: \"\"\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: auth_sync","tags":[]}}}}}}'
-echo
-
-# SOC - Salt Relay logs
-echo
-echo "Setting up SOC - Salt_Relay Logs package policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{"package":{"name":"log","version":"1.1.2"},"name":"soc-salt-relay-logs","namespace":"so","description":"Security Onion - Salt Relay - Logs","policy_id":"so-grid-nodes","inputs":{"logs-logfile":{"enabled":true,"streams":{"log.log":{"enabled":true,"vars":{"paths":["/opt/so/log/soc/salt-relay.log"],"data_stream.dataset":"soc","custom":"pipeline: common","processors": "- dissect:\n    tokenizer: \"%{soc.ts} | %{event.action}\"\n    field: \"message\"\n    target_prefix: \"\"\n- add_fields:\n    target: event\n    fields:\n      category: host\n      module: soc\n      dataset_temp: salt_relay","tags":[]}}}}}}'
-echo

salt/common/tools/sbin/so-elastic-fleet-setup

@@ -1,109 +0,0 @@
-
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0; you may not use
-# this file except in compliance with the Elastic License 2.0.
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-
-. /usr/sbin/so-common
-
-
-# Create ES Token 
-ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' |  jq -r .value)
-printf "ESTOKEN = $ESTOKEN \n"
-
-# Add SO-Manager Fleet URL
-## This array replaces whatever URLs are currently configured
-printf "\n"
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d  '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220"]}'
-printf "\n\n"
-
-# Configure certificates
-mkdir -p /opt/so/conf/elastic-fleet/certs
-cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs
-cp /etc/pki/elasticfleet* /opt/so/conf/elastic-fleet/certs
-
-{% if grains.role in ['so-import', 'so-standalone', 'so-eval', 'so-manager', 'so-managersearch'] %}
-# Add SO-Manager Elasticsearch Ouput
-ESCACRT=$(openssl x509 -in  /opt/so/conf/elastic-fleet/certs/intca.crt)
-JSON_STRING=$( jq -n \
-                  --arg ESCACRT "$ESCACRT" \
-                  '{"name":"so-manager_elasticsearch","id":"so-manager_elasticsearch","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' )
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-printf "\n\n"
-
-{% else %}
-# Create Logstash Output payload
-LOGSTASHCRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/elasticfleet.crt)
-LOGSTASHKEY=$(openssl rsa -in  /opt/so/conf/elastic-fleet/certs/elasticfleet.key)
-LOGSTASHCA=$(openssl x509 -in  /opt/so/conf/elastic-fleet/certs/intca.crt)
-JSON_STRING=$( jq -n \
-                  --arg LOGSTASHCRT "$LOGSTASHCRT" \
-                  --arg LOGSTASHKEY "$LOGSTASHKEY" \
-                  --arg LOGSTASHCA "$LOGSTASHCA" \
-                  '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}'
-                  )
-
-# Add SO-Manager Logstash Ouput
-curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING"
-printf "\n\n"
-{%- endif %}
-
-# Add Elastic Fleet Integrations
-
-# Add Elastic Fleet Server Agent Policy
-#curl -vv -K /opt/so/conf/elasticsearch/curl.config -L \
-#-X POST "localhost:5601/api/fleet/agent_policies" \
-#-H 'kbn-xsrf: true' -H 'Content-Type: application/json' \
-#-d  '{"name":"SO-Manager","id":"so-manager","description":"SO Manager Fleet Server Policy","namespace":"default","monitoring_enabled":["logs"],"has_fleet_server":true}'
-
-# Add Agent Policy - SOS Grid Nodes
-#curl -vv -K /opt/so/conf/elasticsearch/curl.config -L \
-#-X POST "localhost:5601/api/fleet/agent_policies" \
-#-H 'kbn-xsrf: true' -H 'Content-Type: application/json' \
-#-d  '{"name":"SO-Grid","id":"so-grid","description":"SO Grid Endpoint Policy","namespace":"default","monitoring_enabled":["logs"]}'
-
-# Add Agent Policy - Default endpoints
-#curl -vv -K /opt/so/conf/elasticsearch/curl.config -L \
-#-X POST "localhost:5601/api/fleet/agent_policies" \
-#-H 'kbn-xsrf: true' -H 'Content-Type: application/json' \
-#-d  '{"name":"Endpoints-Initalization","id":"endpoints","description":"Initial Endpoint Policy","namespace":"default","monitoring_enabled":["logs"]}'
-
-ENDPOINTSENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("endpoints-default")) | .api_key')
-GRIDNODESENROLLMENTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L "localhost:5601/api/fleet/enrollment_api_keys" -H 'kbn-xsrf: true' -H 'Content-Type: application/json'  | jq .list | jq -r -c '.[] | select(.policy_id | contains("so-grid-nodes")) | .api_key')
-
-# Store needed data in minion pillar
-pillar_file=/opt/so/saltstack/local/pillar/minions/{{ GLOBALS.minion_id }}.sls
-printf '%s\n'\
-    "elasticfleet:"\
-    "  server:"\
-    "    es_token: '$ESTOKEN'"\
-    "    endpoints_enrollment: '$ENDPOINTSENROLLMENTOKEN'"\
-    "    grid_enrollment: '$GRIDNODESENROLLMENTOKEN'"\
-    "    url: '{{ GLOBALS.manager_ip }}'"\
-    "" >> "$pillar_file"
-
-#Store Grid Nodes Enrollment token in Global pillar
-global_pillar_file=/opt/so/saltstack/local/pillar/soc_global.sls
-printf '%s\n'\
-    "  fleet_grid_enrollment_token: '$GRIDNODESENROLLMENTOKEN'"\
-    "" >> "$global_pillar_file"
-
-# Call Elastic-Fleet Salt State
-salt-call state.apply elasticfleet queue=True
-
-# Load Elastic Fleet integrations
-/usr/sbin/so-elastic-fleet-integration-policy-load
-
-# Temp
-wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/  https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.6.2/so-elastic-agent-8.6.2-darwin-x86_64.tar.gz
-wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.6.2/so-elastic-agent-8.6.2-linux-x86_64.tar.gz
-wget --progress=bar:force:noscroll -P /opt/so/saltstack/default/salt/elasticfleet/files/elastic-agent/ https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/so_elastic-agent-8.6.2/so-elastic-agent-8.6.2-windows-x86_64.tar.gz
-
-#git clone -b 2.4-so-elastic-agent https://github.com/Security-Onion-Solutions/securityonion-image.git
-#cd securityonion-image/so-elastic-agent-builder
-#docker build -t so-elastic-agent-builder .
-
-so-elastic-agent-gen-installers
-salt-call state.apply elasticfleet.install_agent_grid queue=True

salt/common/tools/sbin/so-elastic-restart

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%}
-/usr/sbin/so-restart elasticsearch $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
-/usr/sbin/so-restart kibana $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-restart logstash $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-restart curator $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
-/usr/sbin/so-restart elastalert $1
-{%- endif %}

salt/common/tools/sbin/so-elastic-start

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%}
-/usr/sbin/so-start elasticsearch $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
-/usr/sbin/so-start kibana $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-start logstash $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-start curator $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
-/usr/sbin/so-start elastalert $1
-{%- endif %}

salt/common/tools/sbin/so-elastic-stop

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import']%}
-/usr/sbin/so-stop elasticsearch $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval', 'so-manager', 'so-managersearch', 'so-standalone', 'so-import']%}
-/usr/sbin/so-stop kibana $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-stop logstash $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode']%}
-/usr/sbin/so-stop curator $1
-{%- endif %}
-
-{%- if grains['role'] in ['so-eval','so-manager', 'so-managersearch', 'so-standalone']%}
-/usr/sbin/so-stop elastalert $1
-{%- endif %}

salt/common/tools/sbin/so-elasticsearch-ilm-policy-load

@@ -1,21 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-{% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
-{%- set ES_INDEX_SETTINGS = salt['pillar.get']('elasticsearch:index_settings', default=ESCONFIG.elasticsearch.index_settings, merge=True) %}
-{%- set NODEIP = salt['pillar.get']('host:mainip', '') %}
-
-{%- for index, settings in ES_INDEX_SETTINGS.items() %}
-  {%- if settings.policy is defined %}
-echo
-echo "Setting up {{ index }}-logs policy..."
-curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -s -k -L -X PUT "https://{{ NODEIP }}:9200/_ilm/policy/{{ index }}-logs" -H 'Content-Type: application/json' -d'{ "policy": {{ settings.policy | tojson(true) }} }'
-echo
-  {%- endif %}
-{%- endfor %}
-echo

salt/common/tools/sbin/so-elasticsearch-indices-rw

@@ -1,15 +0,0 @@
-#!/bin/bash
-#
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-IP={{ salt['grains.get']('ip_interfaces').get(salt['pillar.get']('sensor:mainint', salt['pillar.get']('manager:mainint', salt['pillar.get']('elasticsearch:mainint', salt['pillar.get']('host:mainint')))))[0] }}
-ESPORT=9200
-
-echo "Removing read only attributes for indices..."
-echo
-curl -K /opt/so/conf/elasticsearch/curl.config -s -k -XPUT -H "Content-Type: application/json" -L https://$IP:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' 2>&1 | if grep -q ack; then echo "Index settings updated..."; else echo "There was any issue updating the read-only attribute.  Please ensure Elasticsearch is running.";fi;

salt/common/tools/sbin/so-firewall

@@ -1,104 +0,0 @@
-#!/usr/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-if [[ $# -lt 1 ]]; then
-  echo "Usage: $0 --role=<ROLE> --ip=<IP ADDRESS> --apply=<true|false>"
-  echo ""
-  echo " Example: so-firewall --role=sensor --ip=192.168.254.100 --apply=true"
-  echo ""
-  exit 1
-fi
-
-for i in "$@"; do
-	case $i in
-		-r=*|--role=*)
-			ROLE="${i#*=}"
-			shift 
-			;;
-		-i=*|--ip=*)
-			IP="${i#*=}"
-			shift 
-			;;
-		-a=*|--apply*)
-			APPLY="${i#*=}"
-			shift
-			;;
-		-*|--*)
-			echo "Unknown option $i"
-			exit 1
-			;;
-		*)
-		;;
-	esac
-done
-
-ROLE=${ROLE,,}
-APPLY=${APPLY,,}
-
-function rolecall() {
-	THEROLE=$1
-	THEROLES="analyst analyst_workstations beats_endpoint beats_endpoint_ssl elastic_agent_endpoint elasticsearch_rest endgame eval heavynodes idh manager managersearch receivers searchnodes sensors standalone strelka_frontend syslog"
-
-	for AROLE in $THEROLES; do
-		if [ "$AROLE" = "$THEROLE" ]; then
-			return 0
-		fi
-	done
-	return 1
-}
-
-# Make sure the required options are specified
-if [ -z "$ROLE" ]; then
-	echo "Please specify a role with --role="
-	exit 1
-fi
-if [ -z "$IP" ]; then
-	echo "Please specify an IP address with --ip="
-	exit 1
-fi
-
-# Are we dealing with a role that this script supports?
-if rolecall "$ROLE"; then
-	echo "$ROLE is a supported role"
-else
-	echo "This is not a supported role"
-	exit 1
-fi
-
- # Are we dealing with an IP?
-if verify_ip4 "$IP"; then
-	echo "$IP is a valid IP or CIDR"
-else
-	echo "$IP is not a valid IP or CIDR"
-	exit 1
-fi
-
-local_salt_dir=/opt/so/saltstack/local/salt/firewall
-
-# Let's see if the file exists and if it does, let's see if the IP exists.
-if [ -f "$local_salt_dir/hostgroups/$ROLE" ]; then
-	if grep -q $IP "$local_salt_dir/hostgroups/$ROLE"; then
-		echo "Host already exists"
-		exit 0
-	fi
-fi
-
-# If you have reached this part of your quest then let's add the IP
-echo "Adding $IP to the $ROLE role"
-echo "$IP" >> $local_salt_dir/hostgroups/$ROLE
-
-# Check to see if we are applying this right away.
-if [ "$APPLY" = "true" ]; then
-	echo "Applying the firewall rules"
-	salt-call state.apply firewall queue=True
-	echo "Firewall rules have been applied... Review logs further if there were errors."
-	echo ""
-else
-	echo "Firewall rules will be applied next salt run"
-fi

salt/common/tools/sbin/so-helix-apikey

@@ -1,27 +0,0 @@
-#!/bin/bash
-
-local_salt_dir=/opt/so/saltstack/local
-
-got_root() {
-
-  # Make sure you are root
-  if [ "$(id -u)" -ne 0 ]; then
-          echo "This script must be run using sudo!"
-          exit 1
-  fi
-
-}
-
-got_root
-if [ ! -f $local_salt_dir/pillar/fireeye/init.sls ]; then
-  echo "This is nto configured for Helix Mode. Please re-install."
-  exit
-else
-  echo "Enter your Helix API Key: "
-  read APIKEY
-  sed -i "s/^    api_key.*/    api_key: $APIKEY/g" $local_salt_dir/pillar/fireeye/init.sls
-  docker stop so-logstash
-  docker rm so-logstash
-  echo "Restarting Logstash for updated key"
-  salt-call state.apply logstash queue=True
-fi

salt/common/tools/sbin/so-image-common

@@ -38,6 +38,7 @@ container_list() {
       "so-zeek"
       "so-elastic-agent"
       "so-elastic-agent-builder"
+      "so-elastic-fleet-package-registry"
     )
   elif [ $MANAGERCHECK != 'so-helix' ]; then
     TRUSTED_CONTAINERS=(
@@ -45,6 +46,7 @@ container_list() {
       "so-elastalert"
       "so-elastic-agent"
       "so-elastic-agent-builder"
+      "so-elastic-fleet-package-registry"
       "so-elasticsearch"
       "so-idh"
       "so-idstools"

salt/common/tools/sbin/so-import-evtx

@@ -1,155 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{%- set MANAGER = salt['grains.get']('master') %}
-{%- set VERSION = salt['pillar.get']('global:soversion') %}
-{%- set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
-{%- set MANAGERIP = salt['pillar.get']('global:managerip') %}
-{%- set URLBASE = salt['pillar.get']('global:url_base') %}
-{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
-{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
-
-INDEX_DATE=$(date +'%Y.%m.%d')
-RUNID=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1)
-LOG_FILE=/nsm/import/evtx-import.log
-
-. /usr/sbin/so-common
-
-function usage {
-  cat << EOF
-Usage: $0 <evtx-file-1> [evtx-file-2] [evtx-file-*]
-
-Imports one or more evtx files into Security Onion. The evtx files will be analyzed and made available for review in the Security Onion toolset.
-EOF
-}
-
-
-function evtx2es() {
-  EVTX=$1
-  HASH=$2
-  
-  docker run --rm \
-    -v "$EVTX:/tmp/data.evtx" \
-    -v "/nsm/import/$HASH/evtx/:/tmp/evtx/" \
-    -v "/nsm/import/evtx-end_newest:/tmp/newest" \
-    -v "/nsm/import/evtx-start_oldest:/tmp/oldest" \
-    --entrypoint "/evtx_calc_timestamps.sh" \
-    {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} >> $LOG_FILE 2>&1
-}
-
-# if no parameters supplied, display usage
-if [ $# -eq 0 ]; then
-  usage
-  exit 1
-fi
-
-# ensure this is a Manager node
-require_manager
-
-# verify that all parameters are files
-for i in "$@"; do
-  if ! [ -f "$i" ]; then
-    usage
-    echo "\"$i\" is not a valid file!"
-    exit 2
-  fi
-done
-
-# track if we have any valid or invalid evtx
-INVALID_EVTXS="no"
-VALID_EVTXS="no"
-
-# track oldest start and newest end so that we can generate the Kibana search hyperlink at the end
-START_OLDEST="2050-12-31"
-END_NEWEST="1971-01-01"
-
-touch /nsm/import/evtx-start_oldest
-touch /nsm/import/evtx-end_newest
-
-echo $START_OLDEST > /nsm/import/evtx-start_oldest
-echo $END_NEWEST > /nsm/import/evtx-end_newest
-
-# paths must be quoted in case they include spaces
-for EVTX in "$@"; do
-  EVTX=$(/usr/bin/realpath "$EVTX")
-  echo "Processing Import: ${EVTX}"
-
-  # generate a unique hash to assist with dedupe checks
-  HASH=$(md5sum "${EVTX}" | awk '{ print $1 }')
-  HASH_DIR=/nsm/import/${HASH}
-  echo "- assigning unique identifier to import: $HASH"
-
-  if [ -d $HASH_DIR ]; then
-    echo "- this EVTX has already been imported; skipping"
-    INVALID_EVTXS="yes"
-  else
-    VALID_EVTXS="yes"
-
-    EVTX_DIR=$HASH_DIR/evtx
-    mkdir -p $EVTX_DIR
-
-    # import evtx and write them to import ingest pipeline
-    echo "- importing logs to Elasticsearch..."
-    evtx2es "${EVTX}" $HASH
-
-    # compare $START to $START_OLDEST
-    START=$(cat /nsm/import/evtx-start_oldest)
-    START_COMPARE=$(date -d $START +%s)
-    START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s)
-    if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then
-      START_OLDEST=$START
-    fi  
-
-    # compare $ENDNEXT to $END_NEWEST
-    END=$(cat /nsm/import/evtx-end_newest)
-    ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"`
-    ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s)
-    END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s)
-    if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then
-      END_NEWEST=$ENDNEXT
-    fi  
-
-    cp -f "${EVTX}" "${EVTX_DIR}"/data.evtx
-    chmod 644 "${EVTX_DIR}"/data.evtx
-
-  fi # end of valid evtx
-
-  echo
-
-done # end of for-loop processing evtx files
-
-# remove temp files
-echo "Cleaning up:"
-for TEMP_EVTX in ${TEMP_EVTXS[@]}; do
-  echo "- removing temporary evtx $TEMP_EVTX"
-  rm -f $TEMP_EVTX
-done
-
-# output final messages
-if [ "$INVALID_EVTXS" = "yes" ]; then
-  echo
-  echo "Please note!  One or more evtx was invalid!  You can scroll up to see which ones were invalid."
-fi
-
-START_OLDEST_FORMATTED=`date +%Y-%m-%d --date="$START_OLDEST"`
-START_OLDEST_SLASH=$(echo $START_OLDEST_FORMATTED | sed -e 's/-/%2F/g')
-END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
-
-if [ "$VALID_EVTXS" = "yes" ]; then
-cat << EOF
-
-Import complete!
-
-You can use the following hyperlink to view data in the time range of your import.  You can triple-click to quickly highlight the entire hyperlink and you can then copy it into your browser:
-https://{{ URLBASE }}/#/dashboards?q=import.id:${RUNID}%20%7C%20groupby%20-sankey%20event.dataset%20event.category%2a%20%7C%20groupby%20-pie%20event.category%20%7C%20groupby%20-bar%20event.module%20%7C%20groupby%20event.dataset%20%7C%20groupby%20event.module%20%7C%20groupby%20event.category%20%7C%20groupby%20observer.name%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC
-
-or you can manually set your Time Range to be (in UTC):
-From: $START_OLDEST_FORMATTED    To: $END_NEWEST
-
-Please note that it may take 30 seconds or more for events to appear in Security Onion Console.
-EOF
-fi

salt/common/tools/sbin/so-logstash-events

@@ -1,17 +0,0 @@
-#!/bin/bash
-#
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% set MAININT = salt['pillar.get']('host:mainint') -%}
-{% set NODEIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] -%}
-
-. /usr/sbin/so-common
-
-if [ "$1" == "" ]; then
-        for i in $(curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines | jq '. | to_entries | .[].key' | sed 's/\"//g'); do echo ${i^}:; curl -s localhost:9600/_node/stats | jq .pipelines.$i.events; done
-else
-        curl -s -L http://{{ NODEIP }}:9600/_node/stats | jq .pipelines.$1.events
-fi

salt/common/tools/sbin/so-minion

@@ -1,304 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-if [ -f /usr/sbin/so-common ]; then
-	. /usr/sbin/so-common
-fi
-
-if [ "$(id -u)" -ne 0 ]; then
-	echo "This script must be run using sudo!"
-	exit 1
-fi
-
-if [[ $# -lt 1 ]]; then
-  echo "Usage: $0 -o=<operation> -m=[id]"
-  echo ""
-  echo " where <operation> is one of the following:"
-  echo ""
-  echo "     list: Lists all keys with hashes"
-  echo "   accept: Accepts a new key and adds the minion files"
-  echo "   delete: Removes the key and deletes the minion files"
-  echo "   reject: Rejects a key"
-  echo "     test: Perform minion test"
-  echo ""
-  exit 1
-fi
-
-for i in "$@"; do
-	case $i in
-		-o=*|--operation=*)
-			OPERATION="${i#*=}"
-			shift 
-			;;
-		-m=*|--minionid=*)
-			MINION_ID="${i#*=}"
-			shift 
-			;;
-		-e=*|--esheap=*)
-			ES_HEAP_SIZE="${i#*=}"
-			shift
-			;;
-		-n=*|--mgmtnic=*)
-			MNIC="${i#*=}"
-			shift
-			;;
-		-d=*|--description=*)
-			NODE_DESCRIPTION="${i#*=}"
-			shift
-			;;
-		-a=*|--monitor=*)
-			INTERFACE="${i#*=}"
-			shift
-			;;
-		-i=*|--ip=*)
-			MAINIP="${i#*=}"
-			shift
-			;;		
-		-*|--*)
-			echo "Unknown option $i"
-			exit 1
-			;;
-		*)
-		;;
-	esac
-done
-
-PILLARFILE=/opt/so/saltstack/local/pillar/minions/$MINION_ID.sls
-ADVPILLARFILE=/opt/so/saltstack/local/pillar/minions/adv_$MINION_ID.sls
-
-function getinstallinfo() {
-	# Pull from file
-	INSTALLVARS=$(sudo salt "$MINION_ID" cp.get_file_str /opt/so/install.txt --out=newline_values_only)
-	source <(echo $INSTALLVARS)
-}
-
-function testminion() {
-	# Always run on the host, since this is going to be the manager of a distributed grid, or an eval/standalone.
-	# Distributed managers must run this in order for the sensor nodes to have access to the so-tcpreplay image.
-	so-test
-	result=$?
-
-	# If this so-minion script is not running on the given minion ID, run so-test remotely on the sensor as well
-	local_id=$(lookup_grain id)
-	if [[ ! "$local_id" =~ "${MINION_ID}_" ]]; then
-		salt "$MINION_ID" cmd.run 'so-test'
-		result=$?
-	fi
-
-	exit $result
-}
-
-function listminions() {
-	salt-key list -F --out=json
-	exit $?
-}
-
-function rejectminion() {
-	salt-key -y -r $MINION_ID
-	exit $?
-}
-
-function acceptminion() {
-	salt-key -y -a $MINION_ID
-}
-
-function deleteminion() {
-	salt-key -y -d $MINION_ID
-}
-
-function deleteminionfiles () {
-	rm -f $PILLARFILE
-	rm -f $ADVPILLARFILE
-}
-
-# Create the minion file
-function create_minion_files() {
-	mkdir -p /opt/so/saltstack/local/pillar/minions
-    touch $ADVPILLARFILE
-	if [ -f "$PILLARFILE" ]; then
-		rm $PILLARFILE
-	fi
-}
-
-# Add Elastic settings to the minion file
-function add_elastic_to_minion() {
-    printf '%s\n'\
-		"elasticsearch:"\
-		"  esheap: '$ES_HEAP_SIZE'"\
-		"  " >> $PILLARFILE
-}
-
-# Add IDH Services info to the minion file
-function add_idh_to_minion() {
-	printf '%s\n'\
-	"idh:"\
-	"  restrict_management_ip: $IDH_MGTRESTRICT"\
-	"  services:" >> "$PILLARFILE"
-	IFS=',' read -ra IDH_SERVICES_ARRAY <<< "$IDH_SERVICES"
-	for service in ${IDH_SERVICES_ARRAY[@]}; do
-	  echo "    - $service" | tr '[:upper:]' '[:lower:]' | tr -d '"' >> "$PILLARFILE"
-	done
-}
-
-function add_logstash_to_minion() {
-	# Create the logstash advanced pillar
-	printf '%s\n'\
-		"logstash_settings:"\
-		"  ls_host: '$LSHOSTNAME'"\
-		"  ls_pipeline_batch_size: 125"\
-		"  ls_input_threads: 1"\
-		"  lsheap: $LSHEAP"\
-		"  ls_pipeline_workers: $CPUCORES"\
-		"  " >> $PILLARFILE
-}
-
-# Analyst Workstation
-function add_analyst_to_minion() {
-    printf '%s\n'\
-		"host:"\
-		"  mainint: '$MNIC'"\
-		"workstation:"\
-		"  gui:"\
-		"    enabled: true"\
-		"sensoroni:"\
-		"  node_description: '${NODE_DESCRIPTION//\'/''}'" >> $PILLARFILE
-}
-
-# Add basic host info to the minion file
-function add_host_to_minion() {
-    printf '%s\n'\
-    "host:"\
-	"  mainip: '$MAINIP'"\
-    "  mainint: '$MNIC'" >> $PILLARFILE
-}
-
-# Add sensoroni specific information - Can we pull node_adrees from the host pillar?
-function add_sensoroni_to_minion() {
-
-	printf '%s\n'\
-    "sensoroni:"\
-		"  node_description: '${NODE_DESCRIPTION//\'/''}'"\
-        "  " >> $PILLARFILE
-}	
-
-# Sensor settings for the minion pillar
-function add_sensor_to_minion() {
-	echo "sensor:" >> $PILLARFILE
-	echo "  interface: '$INTERFACE'" >> $PILLARFILE
-	echo "  mtu: 9000" >> $PILLARFILE
-	echo "zeek:" >> $PILLARFILE
-	echo "    config:" >> $PILLARFILE
-	echo "        node:" >> $PILLARFILE
-	echo "            lb_procs: '$CORECOUNT'" >> $PILLARFILE
-	echo "suricata:" >> $PILLARFILE
-	echo "    config:" >> $PILLARFILE
-	echo "        af-packet:" >> $PILLARFILE
-	echo "            threads: '$CORECOUNT'" >> $PILLARFILE
-	echo "pcap:" >> $PILLARFILE
-	echo "    enabled: True" >> $PILLARFILE
-}
-
-function updateMine() {
-	salt "$MINION_ID" mine.send network.ip_addrs interface="$MNIC"
-}
-function apply_ES_state() {
-	salt-call state.apply elasticsearch concurrent=True
-}
-function createEVAL() {
-	add_elastic_to_minion
-	add_logstash_to_minion
-	add_sensor_to_minion
-}
-
-function createIDH() {
-	add_idh_to_minion
-}
-
-function createIMPORT() {
-	add_elastic_to_minion
-	add_logstash_to_minion
-	add_sensor_to_minion
-}
-
-function createHEAVYNODE() {
-	add_elastic_to_minion
-	add_logstash_to_minion
-	add_sensor_to_minion
-}
-
-function createMANAGER() {
-	add_elastic_to_minion
-	add_logstash_to_minion
-}
-
-function createMANAGERSEARCH() {
-	add_elastic_to_minion
-	add_logstash_to_minion
-}
-
-function createSENSOR() {
-	add_sensor_to_minion
-}
-
-function createSEARCHNODE() {
-	add_elastic_to_minion
-	add_logstash_to_minion
-	updateMine
-	apply_ES_state
-}
-
-function createSTANDALONE() {
-	add_elastic_to_minion
-	add_logstash_to_minion
-	add_sensor_to_minion
-}
-
-function testConnection() {
-	retry 15 3 "salt '$MINION_ID' test.ping" True
-	local ret=$?
-	if [[ $ret != 0 ]]; then
-		echo "The Minion has been accepted but is not online. Try again later"
-		echo "Deleting the key"
-		deleteminion
-		exit 1
-	fi
-}
-
-if [[ "$OPERATION" = 'list' ]]; then
-	listminions
-fi
-
-if [[ "$OPERATION" = 'delete' ]]; then
-	deleteminionfiles
-	deleteminion
-fi
-
-if [[ "$OPERATION" = 'add' || "$OPERATION" = 'setup' ]]; then
-	# Skip this if its setup
-	if [ $OPERATION != 'setup' ]; then
-		# Accept the salt key
-		acceptminion
-		# Test to see if the minion was accepted
-		testConnection
-		# Pull the info from the file to build what is needed
-		getinstallinfo
-	fi
-	# Check to see if nodetype is set
-	if [ -z $NODETYPE ]; then
-		echo "No node type specified"
-		exit 1
-	fi
-	create_minion_files
-	add_host_to_minion
-	add_sensoroni_to_minion
-	create$NODETYPE
-	echo "Minion file created for $MINION_ID"	
-fi
-
-if [[ "$OPERATION" = 'test' ]]; then
-	testminion
-fi

salt/common/tools/sbin/so-nodered-stop

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-/usr/sbin/so-stop nodered $1

salt/common/tools/sbin/so-raid-status

@@ -1,109 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-appliance_check() {
-  {%- if salt['grains.get']('sosmodel', '') %}
-  APPLIANCE=1
-  {%- if grains['sosmodel'] in ['SO2AMI01', 'SO2GCI01', 'SO2AZI01'] %}
-  exit 0
-  {%- endif %}
-  DUDEYOUGOTADELL=$(dmidecode |grep Dell)
-  if [[ -n $DUDEYOUGOTADELL ]]; then
-  APPTYPE=dell
-  else
-  APPTYPE=sm
-  fi
-  mkdir -p /opt/so/log/raid
-
-  {%- else %}
-  echo "This is not an appliance"
-  exit 0
-  {%- endif %}
-}
-
-check_nsm_raid() {
-  PERCCLI=$(/opt/raidtools/perccli/perccli64 /c0/v0 show|grep RAID|grep Optl)
-  MEGACTL=$(/opt/raidtools/megasasctl |grep optimal)
-
-  if [[ $APPLIANCE == '1' ]]; then
-    if [[ -n $PERCCLI ]]; then
-      HWRAID=0
-    elif [[ -n $MEGACTL ]]; then
-      HWRAID=0
-    else
-      HWRAID=1
-    fi
-
-  fi
-
-}
-
-check_boss_raid() {
-  MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional)
-
-  if [[ -n $DUDEYOUGOTADELL ]]; then
-    if [[ -n $MVCLI ]]; then
-      BOSSRAID=0
-    else
-      BOSSRAID=1
-    fi
-  fi
-}
-
-check_software_raid() {
-  if [[ -n $DUDEYOUGOTADELL ]]; then
-    SWRC=$(grep "_" /proc/mdstat)
-
-    if [[ -n $SWRC ]]; then
-        # RAID is failed in some way
-        SWRAID=1
-    else
-        SWRAID=0
-    fi
-  fi
-}
-
-# This script checks raid status if you use SO appliances
-
-# See if this is an appliance
-
-appliance_check
-check_nsm_raid
-check_boss_raid
-{%- if salt['grains.get']('sosmodel', '') %}
-{%- if grains['sosmodel'] in ['SOSMN', 'SOSSNNV'] %}
-check_software_raid
-{%- endif %}
-{%- endif %}
-
-if [[ -n $SWRAID ]]; then
-  if [[ $SWRAID == '0' && $BOSSRAID == '0' ]]; then
-    RAIDSTATUS=0
-  else
-    RAIDSTATUS=1
-  fi
-elif [[ -n $DUDEYOUGOTADELL ]]; then
-  if [[ $BOSSRAID == '0' && $HWRAID == '0' ]]; then
-    RAIDSTATUS=0
-  else
-    RAIDSTATUS=1
-  fi
-elif [[ "$APPTYPE" == 'sm' ]]; then
-  if [[ -n "$HWRAID" ]]; then
-    RAIDSTATUS=0
-  else
-    RAIDSTATUS=1
-  fi
-fi
-
-echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log
-
-

salt/common/tools/sbin/so-restart

@@ -24,6 +24,7 @@ if [ $# -ge 1 ]; then
 
         case $1 in
                 "steno") docker stop so-steno && docker rm so-steno && salt-call state.apply pcap queue=True;;
+                "elastic-fleet") docker stop so-elastic-fleet && docker rm so-elastic-fleet && salt-call state.apply elasticfleet queue=True;;
                 *)  docker stop so-$1 ; docker rm so-$1 ; salt-call state.apply $1 queue=True;;
         esac
 else

salt/common/tools/sbin/so-rule-update

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-. /usr/sbin/so-common
-
-argstr=""
-for arg in "$@"; do
-    argstr="${argstr} \"${arg}\""
-done
-
-docker exec so-idstools /bin/bash -c "cd /opt/so/idstools/etc && idstools-rulecat --force ${argstr}"

salt/common/tools/sbin/so-start

@@ -24,6 +24,7 @@ if [ $# -ge 1 ]; then
 	case $1 in
    		"all") salt-call state.highstate queue=True;;
    		"steno") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply pcap queue=True; fi ;; 
+   		"elastic-fleet") if docker ps | grep -q so-$1; then printf "\n$1 is already running!\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply elasticfleet queue=True; fi ;; 
    		*) if docker ps | grep -E -q '^so-$1$'; then printf "\n$1 is already running\n\n"; else docker rm so-$1 >/dev/null 2>&1 ; salt-call state.apply $1 queue=True; fi ;; 
 	esac
 else

salt/common/tools/sbin/so-status

@@ -103,7 +103,7 @@ def output(options, console, code, data):
 def check_container_status(options, console):
   code = 0
   cli = "docker"
-  proc = subprocess.run([cli, 'ps', '--format', '{{json .}}'], stdout=subprocess.PIPE, encoding="utf-8")
+  proc = subprocess.run([cli, 'ps', '--format', 'json'], stdout=subprocess.PIPE, encoding="utf-8")
   if proc.returncode != 0:
     fail("Container system error; unable to obtain container process statuses")
 
@@ -170,7 +170,8 @@ def main():
   if "-h" in options or "--help" in options or "-?" in options:
     showUsage(options, None)
 
-  if os.environ["USER"] != "root":
+  proc = subprocess.run(['id', '-u'], stdout=subprocess.PIPE, encoding="utf-8")
+  if proc.stdout.strip() != "0":
     fail("This program must be run as root")
   
   console = Console()

salt/common/tools/sbin_jinja/so-desktop-install

@@ -6,17 +6,17 @@
 # Elastic License 2.0.
 
 
-{# we only want the script to install the workstation if it is Rocky -#}
+{# we only want the script to install the desktop if it is Rocky -#}
 {% if grains.os == 'Rocky' -%}
 {#   if this is a manager -#}
 {%   if grains.master == grains.id.split('_')|first -%}
 
 source /usr/sbin/so-common
-doc_workstation_url="$DOC_BASE_URL/analyst-vm.html"
+doc_desktop_url="$DOC_BASE_URL/desktop.html"
 pillar_file="/opt/so/saltstack/local/pillar/minions/{{grains.id}}.sls"
 
 if [ -f "$pillar_file" ]; then
-  if ! grep -q "^workstation:$" "$pillar_file"; then
+  if ! grep -q "^desktop:$" "$pillar_file"; then
 
     FIRSTPASS=yes
     while [[ $INSTALL != "yes" ]] && [[ $INSTALL != "no" ]]; do
@@ -26,7 +26,7 @@ if [ -f "$pillar_file" ]; then
         echo "##    _______________________________    ##"
         echo "##                                       ##"
         echo "##    Installing the Security Onion      ##"
-        echo "##   analyst node on this device will    ##"
+        echo "##     Desktop on this device will       ##"
         echo "##       make permanent changes to       ##"
         echo "##              the system.              ##"
         echo "##    A system reboot will be required   ##"
@@ -42,40 +42,40 @@ if [ -f "$pillar_file" ]; then
     done
 
     if [[ $INSTALL == "no" ]]; then
-      echo "Exiting analyst node installation."
+      echo "Exiting desktop node installation."
       exit 0
     fi
 
-    # Add workstation pillar to the minion's pillar file
+    # Add desktop pillar to the minion's pillar file
     printf '%s\n'\
-      "workstation:"\
+      "desktop:"\
       "  gui:"\
       "    enabled: true"\
 		  "" >> "$pillar_file"
-    echo "Applying the workstation state. This could take some time since there are many packages that need to be installed."
+    echo "Applying the desktop state. This could take some time since there are many packages that need to be installed."
-    if salt-call state.apply workstation -linfo queue=True; then # make sure the state ran successfully
+    if salt-call state.apply desktop -linfo queue=True; then # make sure the state ran successfully
       echo ""
-      echo "Analyst workstation has been installed!"
+      echo "Security Onion Desktop has been installed!"
       echo "Press ENTER to reboot or Ctrl-C to cancel."
       read pause
 
       reboot;
     else
-      echo "There was an issue applying the workstation state. Please review the log above or at /opt/so/logs/salt/minion."
+      echo "There was an issue applying the desktop state. Please review the log above or at /opt/so/log/salt/minion."
     fi
-  else # workstation is already added
+  else # desktop is already added
-    echo "The workstation pillar already exists in $pillar_file."
+    echo "The desktop pillar already exists in $pillar_file."
-    echo "To enable/disable the gui, set 'workstation:gui:enabled' to true or false in $pillar_file."
+    echo "To enable/disable the gui, set 'desktop:gui:enabled' to true or false in $pillar_file."
-    echo "Additional documentation can be found at $doc_workstation_url."
+    echo "Additional documentation can be found at $doc_desktop_url."
   fi
 else # if the pillar file doesn't exist
-  echo "Could not find $pillar_file and add the workstation pillar."
+  echo "Could not find $pillar_file and add the desktop pillar."
 fi
 
 {#-  if this is not a manager #}
 {%   else -%}
 
-echo "Since this is not a manager, the pillar values to enable analyst workstation must be set manually. Please view the documentation at $doc_workstation_url."
+echo "Since this is not a manager, the pillar values to enable Security Onion Desktop must be set manually. Please view the documentation at $doc_desktop_url."
 
 {#- endif if this is a manager #}
 {%   endif -%}
@@ -83,7 +83,7 @@ echo "Since this is not a manager, the pillar values to enable analyst workstati
 {#- if not Rocky #}
 {%- else %}
 
-echo "The Analyst Workstation can only be installed on Rocky. Please view the documentation at $doc_workstation_url."
+echo "The Security Onion Desktop can only be installed on Rocky Linux. Please view the documentation at $doc_desktop_url."
 
 {#- endif grains.os == Rocky #}
 {% endif -%}

salt/common/tools/sbin_jinja/so-import-evtx

@@ -0,0 +1,235 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{%- set MANAGER = salt['grains.get']('master') %}
+{%- set VERSION = salt['pillar.get']('global:soversion') %}
+{%- set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
+{%- set MANAGERIP = salt['pillar.get']('global:managerip') %}
+{%- set URLBASE = salt['pillar.get']('global:url_base') %}
+{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
+{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
+
+INDEX_DATE=$(date +'%Y.%m.%d')
+LOG_FILE=/nsm/import/evtx-import.log
+
+. /usr/sbin/so-common
+
+function usage {
+  cat << EOF
+Usage: $0 [options] <evtx-file-1> [evtx-file-2] [evtx-file-*]
+
+Imports one or more evtx files into Security Onion. The evtx files will be analyzed and made available for review in the Security Onion toolset.
+
+Options:
+  --json      Outputs summary in JSON format. Implies --quiet.
+  --quiet     Silences progress information to stdout.
+  --shift     Adds a time shift. Accepts a single argument that is intended to be the date of the last record, and shifts the dates of the previous records accordingly.
+              Ex. sudo so-import-evtx --shift "2023-08-01 01:01:01" example.evtx 
+EOF
+}
+
+quiet=0
+json=0
+INPUT_FILES=
+while [[ $# -gt 0 ]]; do
+  param=$1
+  shift
+  case "$param" in
+    --json)
+      json=1
+      quiet=1
+      ;;
+    --quiet)
+      quiet=1
+      ;;
+    --shift)
+      SHIFTDATE=$1
+      shift
+      ;;
+    -*) 
+      echo "Encountered unexpected parameter: $param"
+      usage
+      exit 1
+      ;;
+    *) 
+      if [[ "$INPUT_FILES" != "" ]]; then
+        INPUT_FILES="$INPUT_FILES $param"
+      else
+        INPUT_FILES="$param"
+      fi
+      ;;
+  esac
+done
+
+function status {
+  msg=$1
+  [[ $quiet -eq 1 ]] && return
+  echo "$msg"
+}
+
+function evtx2es() {
+  EVTX=$1
+  HASH=$2
+  SHIFTDATE=$3
+  
+  docker run --rm \
+    -e "SHIFTTS=$SHIFTDATE" \
+    -v "$EVTX:/tmp/data.evtx" \
+    -v "/nsm/import/$HASH/evtx/:/tmp/evtx/" \
+    -v "/nsm/import/evtx-end_newest:/tmp/newest" \
+    -v "/nsm/import/evtx-start_oldest:/tmp/oldest" \
+    --entrypoint "/evtx_calc_timestamps.sh" \
+    {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} >> $LOG_FILE 2>&1
+}
+
+# if no parameters supplied, display usage
+if [ "$INPUT_FILES" == "" ]; then
+  usage
+  exit 1
+fi
+
+# ensure this is a Manager node
+require_manager @> /dev/null
+
+# verify that all parameters are files
+for i in $INPUT_FILES; do
+  if ! [ -f "$i" ]; then
+    echo "\"$i\" is not a valid file!"
+    exit 2
+  fi
+done
+
+# track oldest start and newest end so that we can generate the Kibana search hyperlink at the end
+START_OLDEST="2050-12-31"
+END_NEWEST="1971-01-01"
+
+INVALID_EVTXS_COUNT=0
+VALID_EVTXS_COUNT=0
+SKIPPED_EVTXS_COUNT=0
+
+touch /nsm/import/evtx-start_oldest
+touch /nsm/import/evtx-end_newest
+
+echo $START_OLDEST > /nsm/import/evtx-start_oldest
+echo $END_NEWEST > /nsm/import/evtx-end_newest
+
+# paths must be quoted in case they include spaces
+for EVTX in $INPUT_FILES; do
+  EVTX=$(/usr/bin/realpath "$EVTX")
+  status "Processing Import: ${EVTX}"
+  if ! [ -z "$SHIFTDATE" ]; then
+    status "- timeshifting logs to end date of $SHIFTDATE"
+  fi
+  # generate a unique hash to assist with dedupe checks
+  HASH=$(md5sum "${EVTX}" | awk '{ print $1 }')
+  HASH_DIR=/nsm/import/${HASH}
+  status "- assigning unique identifier to import: $HASH"
+
+  if [[ "$HASH_FILTERS" == "" ]]; then
+    HASH_FILTERS="import.id:${HASH}"
+    HASHES="${HASH}"
+  else
+    HASH_FILTERS="$HASH_FILTERS%20OR%20import.id:${HASH}"
+    HASHES="${HASHES} ${HASH}"
+  fi
+
+  if [ -d $HASH_DIR ]; then
+    status "- this EVTX has already been imported; skipping"
+    SKIPPED_EVTXS_COUNT=$((SKIPPED_EVTXS_COUNT + 1))
+  else
+    EVTX_DIR=$HASH_DIR/evtx
+    mkdir -p $EVTX_DIR
+
+    # import evtx and write them to import ingest pipeline
+    status "- importing logs to Elasticsearch..."
+    evtx2es "${EVTX}" $HASH "$SHIFTDATE"
+    if [[ $? -ne 0 ]]; then
+      INVALID_EVTXS_COUNT=$((INVALID_EVTXS_COUNT + 1))
+      status "- WARNING: This evtx file may not have fully imported successfully"
+    else
+      VALID_EVTXS_COUNT=$((VALID_EVTXS_COUNT + 1))
+    fi
+
+    # compare $START to $START_OLDEST
+    START=$(cat /nsm/import/evtx-start_oldest)
+    START_COMPARE=$(date -d $START +%s)
+    START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s)
+    if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then
+      START_OLDEST=$START
+    fi  
+
+    # compare $ENDNEXT to $END_NEWEST
+    END=$(cat /nsm/import/evtx-end_newest)
+    ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"`
+    ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s)
+    END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s)
+    if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then
+      END_NEWEST=$ENDNEXT
+    fi  
+
+    cp -f "${EVTX}" "${EVTX_DIR}"/data.evtx
+    chmod 644 "${EVTX_DIR}"/data.evtx
+
+  fi # end of valid evtx
+
+  status
+
+done # end of for-loop processing evtx files
+
+# output final messages
+if [[ $INVALID_EVTXS_COUNT -gt 0 ]]; then
+  status
+  status "Please note!  One or more evtx was invalid!  You can scroll up to see which ones were invalid."
+fi
+
+START_OLDEST_FORMATTED=`date +%Y-%m-%d --date="$START_OLDEST"`
+START_OLDEST_SLASH=$(echo $START_OLDEST_FORMATTED | sed -e 's/-/%2F/g')
+END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
+
+if [[ $VALID_EVTXS_COUNT -gt 0 ]] || [[ $SKIPPED_EVTXS_COUNT -gt 0 ]]; then
+  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20-sankey%20event.dataset%20event.category%2a%20%7C%20groupby%20-pie%20event.category%20%7C%20groupby%20-bar%20event.module%20%7C%20groupby%20event.dataset%20%7C%20groupby%20event.module%20%7C%20groupby%20event.category%20%7C%20groupby%20observer.name%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
+
+  status "Import complete!"
+  status
+  status "Use the following hyperlink to view the imported data. Triple-click to quickly highlight the entire hyperlink and then copy it into a browser:"
+  status
+  status "$URL"
+  status
+  status "or, manually set the Time Range to be (in UTC):"
+  status
+  status "From: $START_OLDEST_FORMATTED    To: $END_NEWEST"
+  status
+  status "Note: It can take 30 seconds or more for events to appear in Security Onion Console."
+  RESULT=0
+else
+  START_OLDEST=
+  END_NEWEST=
+  URL=
+  RESULT=1
+fi
+
+if [[ $json -eq 1 ]]; then
+  jq -n \
+    --arg success_count "$VALID_EVTXS_COUNT" \
+    --arg fail_count "$INVALID_EVTXS_COUNT" \
+    --arg skipped_count "$SKIPPED_EVTXS_COUNT" \
+    --arg begin_date "$START_OLDEST" \
+    --arg end_date "$END_NEWEST" \
+    --arg url "$URL" \
+    --arg hashes "$HASHES" \
+    '''{
+      success_count: $success_count,
+      fail_count: $fail_count,
+      skipped_count: $skipped_count,
+      begin_date: $begin_date,
+      end_date: $end_date,
+      url: $url,
+      hash: ($hashes / " ")
+    }'''
+fi
+
+exit $RESULT

salt/common/tools/sbin_jinja/so-import-pcap

@@ -15,12 +15,51 @@
 
 function usage {
   cat << EOF
-Usage: $0 <pcap-file-1> [pcap-file-2] [pcap-file-N]
+Usage: $0 [options] <pcap-file-1> [pcap-file-2] [pcap-file-N]
 
 Imports one or more PCAP files onto a sensor node. The PCAP traffic will be analyzed and made available for review in the Security Onion toolset.
+
+Options:
+  --json      Outputs summary in JSON format. Implies --quiet.
+  --quiet     Silences progress information to stdout.
 EOF
 }
 
+quiet=0
+json=0
+INPUT_FILES=
+while [[ $# -gt 0 ]]; do
+  param=$1
+  shift
+  case "$param" in
+    --json)
+      json=1
+      quiet=1
+      ;;
+    --quiet)
+      quiet=1
+      ;;
+    -*) 
+      echo "Encountered unexpected parameter: $param"
+      usage
+      exit 1
+      ;;
+    *) 
+      if [[ "$INPUT_FILES" != "" ]]; then
+        INPUT_FILES="$INPUT_FILES $param"
+      else
+        INPUT_FILES="$param"
+      fi
+      ;;
+  esac
+done
+
+function status {
+  msg=$1
+  [[ $quiet -eq 1 ]] && return
+  echo "$msg"
+}
+
 function pcapinfo() {
   PCAP=$1
   ARGS=$2
@@ -84,7 +123,7 @@ function zeek() {
 }
 
 # if no parameters supplied, display usage
-if [ $# -eq 0 ]; then
+if [ "$INPUT_FILES" == "" ]; then
   usage
   exit 1
 fi
@@ -96,31 +135,30 @@ if [ ! -d /opt/so/conf/suricata ]; then
 fi
 
 # verify that all parameters are files
-for i in "$@"; do
+for i in $INPUT_FILES; do
   if ! [ -f "$i" ]; then
-    usage
     echo "\"$i\" is not a valid file!"
     exit 2
   fi
 done
 
-# track if we have any valid or invalid pcaps
-INVALID_PCAPS="no"
-VALID_PCAPS="no"
-
 # track oldest start and newest end so that we can generate the Kibana search hyperlink at the end
 START_OLDEST="2050-12-31"
 END_NEWEST="1971-01-01"
 
+INVALID_PCAPS_COUNT=0
+VALID_PCAPS_COUNT=0
+SKIPPED_PCAPS_COUNT=0
+
 # paths must be quoted in case they include spaces
-for PCAP in "$@"; do
+for PCAP in $INPUT_FILES; do
   PCAP=$(/usr/bin/realpath "$PCAP")
-  echo "Processing Import: ${PCAP}"
+  status "Processing Import: ${PCAP}"
-  echo "- verifying file"
+  status "- verifying file"
   if ! pcapinfo "${PCAP}" > /dev/null 2>&1; then
     # try to fix pcap and then process the fixed pcap directly
     PCAP_FIXED=`mktemp /tmp/so-import-pcap-XXXXXXXXXX.pcap`
-    echo "- attempting to recover corrupted PCAP file"
+    status "- attempting to recover corrupted PCAP file"
     pcapfix "${PCAP}" "${PCAP_FIXED}"
     # Make fixed file world readable since the Suricata docker container will runas a non-root user
     chmod a+r "${PCAP_FIXED}"
@@ -131,33 +169,44 @@ for PCAP in "$@"; do
   # generate a unique hash to assist with dedupe checks
   HASH=$(md5sum "${PCAP}" | awk '{ print $1 }')
   HASH_DIR=/nsm/import/${HASH}
-  echo "- assigning unique identifier to import: $HASH"
+  status "- assigning unique identifier to import: $HASH"
 
-  if [ -d $HASH_DIR ]; then
+  pcap_data=$(pcapinfo "${PCAP}")
-    echo "- this PCAP has already been imported; skipping"
+  if ! echo "$pcap_data" | grep -q "First packet time:" || echo "$pcap_data" |egrep -q "Last packet time:    1970-01-01|Last packet time:    n/a"; then
-    INVALID_PCAPS="yes"
+    status "- this PCAP file is invalid; skipping"
-  elif pcapinfo "${PCAP}" |egrep -q "Last packet time:    1970-01-01|Last packet time:    n/a"; then
+    INVALID_PCAPS_COUNT=$((INVALID_PCAPS_COUNT + 1))
-    echo "- this PCAP file is invalid; skipping"
-    INVALID_PCAPS="yes"
   else
-    VALID_PCAPS="yes"
+    if [ -d $HASH_DIR ]; then
+      status "- this PCAP has already been imported; skipping"
+      SKIPPED_PCAPS_COUNT=$((SKIPPED_PCAPS_COUNT + 1))
+    else
+      VALID_PCAPS_COUNT=$((VALID_PCAPS_COUNT + 1))
 
-    PCAP_DIR=$HASH_DIR/pcap
+      PCAP_DIR=$HASH_DIR/pcap
-    mkdir -p $PCAP_DIR
+      mkdir -p $PCAP_DIR
 
-    # generate IDS alerts and write them to standard pipeline
+      # generate IDS alerts and write them to standard pipeline
-    echo "- analyzing traffic with Suricata"
+      status "- analyzing traffic with Suricata"
-    suricata "${PCAP}" $HASH
+      suricata "${PCAP}" $HASH
-    {% if salt['pillar.get']('global:mdengine') == 'ZEEK' %}
+      {% if salt['pillar.get']('global:mdengine') == 'ZEEK' %}
-    # generate Zeek logs and write them to a unique subdirectory in /nsm/import/zeek/
+      # generate Zeek logs and write them to a unique subdirectory in /nsm/import/zeek/
-    # since each run writes to a unique subdirectory, there is no need for a lock file
+      # since each run writes to a unique subdirectory, there is no need for a lock file
-    echo "- analyzing traffic with Zeek"
+      status "- analyzing traffic with Zeek"
-    zeek "${PCAP}" $HASH
+      zeek "${PCAP}" $HASH
-    {% endif %}
+      {% endif %}
+    fi
+
+    if [[ "$HASH_FILTERS" == "" ]]; then
+      HASH_FILTERS="import.id:${HASH}"
+      HASHES="${HASH}"
+    else
+      HASH_FILTERS="$HASH_FILTERS%20OR%20import.id:${HASH}"
+      HASHES="${HASHES} ${HASH}"
+    fi
 
     START=$(pcapinfo "${PCAP}" -a |grep "First packet time:" | awk '{print $4}')
     END=$(pcapinfo "${PCAP}" -e |grep "Last packet time:" | awk '{print $4}')
-    echo "- saving PCAP data spanning dates $START through $END"
+    status "- found PCAP data spanning dates $START through $END"
 
     # compare $START to $START_OLDEST
     START_COMPARE=$(date -d $START +%s)
@@ -179,37 +228,62 @@ for PCAP in "$@"; do
 
   fi # end of valid pcap
 
-  echo
+  status
 
 done # end of for-loop processing pcap files
 
 # remove temp files
-echo "Cleaning up:"
 for TEMP_PCAP in ${TEMP_PCAPS[@]}; do
-  echo "- removing temporary pcap $TEMP_PCAP"
+  status "- removing temporary pcap $TEMP_PCAP"
   rm -f $TEMP_PCAP
 done
 
 # output final messages
-if [ "$INVALID_PCAPS" = "yes" ]; then
+if [[ $INVALID_PCAPS_COUNT -gt 0 ]]; then
-  echo
+  status
-  echo "Please note!  One or more pcaps was invalid!  You can scroll up to see which ones were invalid."
+  status "WARNING: One or more pcaps was invalid. Scroll up to see which ones were invalid."
 fi
 
 START_OLDEST_SLASH=$(echo $START_OLDEST | sed -e 's/-/%2F/g')
 END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
+if [[ $VALID_PCAPS_COUNT -gt 0 ]] || [[ $SKIPPED_PCAPS_COUNT -gt 0 ]]; then
+  URL="https://{{ URLBASE }}/#/dashboards?q=$HASH_FILTERS%20%7C%20groupby%20-sankey%20event.dataset%20event.category%2a%20%7C%20groupby%20-pie%20event.category%20%7C%20groupby%20-bar%20event.module%20%7C%20groupby%20event.dataset%20%7C%20groupby%20event.module%20%7C%20groupby%20event.category%20%7C%20groupby%20observer.name%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC"
 
-if [ "$VALID_PCAPS" = "yes" ]; then
+  status "Import complete!"
-cat << EOF
+  status
-
+  status "Use the following hyperlink to view the imported data. Triple-click to quickly highlight the entire hyperlink and then copy it into a browser:"
-Import complete!
+  status "$URL"
-
+  status
-You can use the following hyperlink to view data in the time range of your import.  You can triple-click to quickly highlight the entire hyperlink and you can then copy it into your browser:
+  status "or, manually set the Time Range to be (in UTC):"
-https://{{ URLBASE }}/#/dashboards?q=import.id:${HASH}%20%7C%20groupby%20-sankey%20event.dataset%20event.category%2a%20%7C%20groupby%20-pie%20event.category%20%7C%20groupby%20-bar%20event.module%20%7C%20groupby%20event.dataset%20%7C%20groupby%20event.module%20%7C%20groupby%20event.category%20%7C%20groupby%20observer.name%20%7C%20groupby%20source.ip%20%7C%20groupby%20destination.ip%20%7C%20groupby%20destination.port&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC
+  status "From: $START_OLDEST    To: $END_NEWEST"
-
+  status
-or you can manually set your Time Range to be (in UTC):
+  status "Note: It can take 30 seconds or more for events to appear in Security Onion Console."
-From: $START_OLDEST    To: $END_NEWEST
+  RESULT=0
-
+else
-Please note that it may take 30 seconds or more for events to appear in Security Onion Console.
+  START_OLDEST=
-EOF
+  END_NEWEST=
+  URL=
+  RESULT=1
 fi
+
+if [[ $json -eq 1 ]]; then
+  jq -n \
+    --arg success_count "$VALID_PCAPS_COUNT" \
+    --arg fail_count "$INVALID_PCAPS_COUNT" \
+    --arg skipped_count "$SKIPPED_PCAPS_COUNT" \
+    --arg begin_date "$START_OLDEST" \
+    --arg end_date "$END_NEWEST" \
+    --arg url "$URL" \
+    --arg hashes "$HASHES" \
+    '''{
+      success_count: $success_count,
+      fail_count: $fail_count,
+      skipped_count: $skipped_count,
+      begin_date: $begin_date,
+      end_date: $end_date,
+      url: $url,
+      hash: ($hashes / " ")
+    }'''
+fi
+
+exit $RESULT

salt/common/tools/sbin_jinja/so-raid-status

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+
+
+. /usr/sbin/so-common
+
+{%- if salt['grains.get']('sosmodel', '') %}
+{%- set model = salt['grains.get']('sosmodel') %}
+model={{ model }}
+# Don't need cloud images to use this
+if [[ $model =~ ^(SO2AMI01|SO2AZI01|SO2GCI01)$ ]]; then
+    exit 0
+fi
+{%- else %}
+echo "This is not an appliance"
+exit 0
+{%- endif %}
+if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200|SOSSNNV|SOSMN)$ ]]; then
+	is_bossraid=true
+fi
+if [[ $model =~ ^(SOSSNNV|SOSMN)$ ]]; then
+	is_swraid=true
+fi
+if [[ $model =~ ^(SOS10K|SOS500|SOS1000|SOS1000F|SOS4000|SOSSN7200)$ ]]; then
+	is_hwraid=true
+fi
+
+check_nsm_raid() {
+  PERCCLI=$(/opt/raidtools/perccli/perccli64 /c0/v0 show|grep RAID|grep Optl)
+  MEGACTL=$(/opt/raidtools/megasasctl |grep optimal)
+
+  if [[ $APPLIANCE == '1' ]]; then
+    if [[ -n $PERCCLI ]]; then
+      HWRAID=0
+    elif [[ -n $MEGACTL ]]; then
+      HWRAID=0
+    else
+      HWRAID=1
+    fi
+
+  fi
+
+}
+
+check_boss_raid() {
+  MVCLI=$(/usr/local/bin/mvcli info -o vd |grep status |grep functional)
+
+  if [[ -n $MVCLI ]]; then
+    BOSSRAID=0
+  else
+    BOSSRAID=1
+  fi
+}
+
+check_software_raid() {
+  SWRC=$(grep "_" /proc/mdstat)
+  if [[ -n $SWRC ]]; then
+      # RAID is failed in some way
+      SWRAID=1
+  else
+      SWRAID=0
+  fi
+}
+
+# Set everything to 0
+SWRAID=0
+BOSSRAID=0
+HWRAID=0
+
+if [[ $is_hwraid ]]; then
+	check_nsm_raid
+fi
+if [[ $is_bossraid ]]; then
+	check_boss_raid
+fi
+if [[ $is_swraid ]]; then
+	check_software_raid
+fi
+
+sum=$(($SWRAID + $BOSSRAID + $HWRAID))
+
+if [[ $sum == "0" ]]; then
+  RAIDSTATUS=0
+else
+  RAIDSTATUS=1
+fi
+
+echo "nsmraid=$RAIDSTATUS" > /opt/so/log/raid/status.log

salt/curator/config.sls

@@ -0,0 +1,81 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from "curator/map.jinja" import CURATORMERGED %}
+
+# Create the group
+curatorgroup:
+  group.present:
+    - name: curator
+    - gid: 934
+
+# Add user
+curator:
+  user.present:
+    - uid: 934
+    - gid: 934
+    - home: /opt/so/conf/curator
+    - createhome: False
+
+# Create the log directory
+curlogdir:
+  file.directory:
+    - name: /opt/so/log/curator
+    - user: 934
+    - group: 939
+
+curactiondir:
+  file.directory:
+    - name: /opt/so/conf/curator/action
+    - user: 934
+    - group: 939
+    - makedirs: True
+
+actionconfs:
+  file.recurse:
+    - name: /opt/so/conf/curator/action
+    - source: salt://curator/files/action
+    - user: 934
+    - group: 939
+    - template: jinja
+    - defaults:
+        CURATORMERGED: {{ CURATORMERGED.elasticsearch.index_settings }}
+        
+curconf:
+  file.managed:
+    - name: /opt/so/conf/curator/curator.yml
+    - source: salt://curator/files/curator.yml
+    - user: 934
+    - group: 939
+    - mode: 660
+    - template: jinja
+    - show_changes: False
+
+curator_sbin:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://curator/tools/sbin
+    - user: 934
+    - group: 939
+    - file_mode: 755
+
+curator_sbin_jinja:
+  file.recurse:
+    - name: /usr/sbin
+    - source: salt://curator/tools/sbin_jinja
+    - user: 934
+    - group: 939 
+    - file_mode: 755
+    - template: jinja
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/curator/defaults.yaml

@@ -1,98 +1,100 @@
-elasticsearch:
+curator:
-  index_settings:
+  enabled: False
-    logs-import-so:
+  elasticsearch:
-      close: 73000
+    index_settings:
-      delete: 73001
+      logs-import-so:
-    logs-strelka-so:
+        close: 73000
-      close: 30
+        delete: 73001
-      delete: 365
+      logs-strelka-so:
-    logs-suricata-so:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-suricata-so:
-    logs-syslog-so:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-syslog-so:
-    logs-zeek-so:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-zeek-so:
-    logs-elastic_agent-metricbeat-default:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-elastic_agent-metricbeat-default:
-    logs-elastic_agent-osquerybeat-default:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-elastic_agent-osquerybeat-default:
-    logs-elastic_agent-fleet_server-default:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-elastic_agent-fleet_server-default:
-    logs-elastic_agent-filebeat-default:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-elastic_agent-filebeat-default:
-    logs-elastic_agent-default:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-elastic_agent-default:
-    logs-system-auth-default:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-system-auth-default:
-    logs-system-application-default:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-system-application-default:
-    logs-system-security-default:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-system-security-default:
-    logs-system-system-default:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-system-system-default:
-    logs-system-syslog-default:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-system-syslog-default:
-    logs-windows-powershell-default:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-windows-powershell-default:
-    logs-windows-sysmon_operational-default:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      logs-windows-sysmon_operational-default:
-    so-beats:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      so-beats:
-    so-elasticsearch:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      so-elasticsearch:
-    so-firewall:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      so-firewall:
-    so-ids:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      so-ids:
-    so-import:
+        close: 30
-      close: 73000
+        delete: 365
-      delete: 73001
+      so-import:
-    so-kratos:
+        close: 73000
-      close: 30
+        delete: 73001
-      delete: 365
+      so-kratos:
-    so-kibana:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      so-kibana:
-    so-logstash:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      so-logstash:
-    so-netflow:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      so-netflow:
-    so-osquery:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      so-osquery:
-    so-ossec:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      so-ossec:
-    so-redis:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      so-redis:
-    so-strelka:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      so-strelka:
-    so-syslog:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      so-syslog:
-    so-zeek:
+        close: 30
-      close: 30
+        delete: 365
-      delete: 365
+      so-zeek:
+        close: 30
+        delete: 365

salt/curator/disabled.sls

@@ -0,0 +1,35 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+
+include:
+  - curator.sostatus
+  
+so-curator:
+  docker_container.absent:
+    - force: True
+
+so-curator_so-status.disabled:
+  file.comment:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - regex: ^so-curator$
+
+so-curator-cluster-close:
+  cron.absent:
+    - identifier: so-curator-cluster-close
+
+so-curator-cluster-delete:
+  cron.absent:
+    - identifier: so-curator-cluster-delete
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/curator/enabled.sls

@@ -0,0 +1,88 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+{%   from 'vars/globals.map.jinja' import GLOBALS %}
+{%   from 'docker/docker.map.jinja' import DOCKER %}
+
+include:
+  - curator.config
+  - curator.sostatus
+
+so-curator:
+  docker_container.running:
+    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-curator:{{ GLOBALS.so_version }}
+    - start: True
+    - hostname: curator
+    - name: so-curator
+    - user: curator
+    - networks:
+      - sobridge:
+        - ipv4_address: {{ DOCKER.containers['so-curator'].ip }}
+    - interactive: True
+    - tty: True
+    - binds:
+      - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro
+      - /opt/so/conf/curator/action/:/etc/curator/action:ro
+      - /opt/so/log/curator:/var/log/curator:rw
+      {% if DOCKER.containers['so-curator'].custom_bind_mounts %}
+        {% for BIND in DOCKER.containers['so-curator'].custom_bind_mounts %}
+      - {{ BIND }}
+        {% endfor %}
+      {% endif %}
+      {% if DOCKER.containers['so-curator'].extra_hosts %}
+    - extra_hosts:
+        {% for XTRAHOST in DOCKER.containers['so-curator'].extra_hosts %}
+      - {{ XTRAHOST }}
+        {% endfor %}
+      {% endif %}
+      {% if DOCKER.containers['so-curator'].extra_env %}
+    - environment:
+        {% for XTRAENV in DOCKER.containers['so-curator'].extra_env %}
+      - {{ XTRAENV }}
+        {% endfor %}
+      {% endif %}
+    - require:
+      - file: actionconfs
+      - file: curconf
+      - file: curlogdir
+    - watch:
+      - file: curconf
+
+delete_so-curator_so-status.disabled:
+  file.uncomment:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - regex: ^so-curator$
+
+so-curator-cluster-close:
+  cron.present:
+    - name: /usr/sbin/so-curator-cluster-close > /opt/so/log/curator/cron-close.log 2>&1
+    - identifier: so-curator-cluster-close
+    - user: root
+    - minute: '2'
+    - hour: '*/1'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
+so-curator-cluster-delete:
+  cron.present:
+    - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-cluster-delete.log 2>&1
+    - identifier: so-curator-cluster-delete
+    - user: root
+    - minute: '*/5'
+    - hour: '*'
+    - daymonth: '*'
+    - month: '*'
+    - dayweek: '*'
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}

salt/curator/files/action/logs-elastic_agent-default-close.yaml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-elastic_agent-filebeat-default-close.yaml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-elastic_agent-fleet_server-default-close.yaml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-elastic_agent-metricbeat-default-close.yaml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-elastic_agent-osquerybeat-default-close.yaml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-import-so-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-strelka-so-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-suricata-so-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-syslog-so-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-system-application-default-close.yaml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-system-auth-default-close.yaml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-system-security-default-close.yaml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-system-syslog-default-close.yaml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-system-system-default-close.yaml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-windows-powershell-default-close.yaml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-windows-sysmon_operational-default-close.yaml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/logs-zeek-so-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-beats-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-elasticsearch-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-firewall-close.yml

@@ -13,7 +13,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-ids-close.yml

@@ -13,7 +13,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-import-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-kibana-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-kratos-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-logstash-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-netflow-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-osquery-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-ossec-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-redis-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-strelka-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-syslog-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/action/so-zeek-close.yml

@@ -12,7 +12,7 @@ actions:
     options:
       delete_aliases: False
       timeout_override:
-      continue_if_exception: False
+      ignore_empty_list: True
       disable_action: False
     filters:
     - filtertype: pattern

salt/curator/files/bin/so-curator-cluster-delete-delete

@@ -1,85 +0,0 @@
-#!/bin/bash
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% import_yaml 'elasticsearch/defaults.yaml' as ELASTICDEFAULTS %}
-{%- set ELASTICSEARCH_HOST = GLOBALS.node_ip -%}
-{%- set RETENTION = salt['pillar.get']('elasticsearch:retention', ELASTICDEFAULTS.elasticsearch.retention, merge=true) -%}
-
-LOG="/opt/so/log/curator/so-curator-cluster-delete.log"
-LOG_SIZE_LIMIT=$(/usr/sbin/so-elasticsearch-cluster-space-total {{ RETENTION.retention_pct}})
-
-overlimit() {
-         [[ $(/usr/sbin/so-elasticsearch-cluster-space-used) -gt "${LOG_SIZE_LIMIT}" ]]
-}
-
-closedindices() {
-	# If we can't query Elasticsearch, then immediately return false.
-        /usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep close > /dev/null 2>&1
-        [ $? -eq 1 ] && return false 
-	# First, get the list of closed indices using _cat/indices?h=index,status | grep close | awk '{print $1}'.
-	# Next, filter out any so-case indices.
-	# Finally, use grep's -q option to return true if there are any remaining logstash-, so-, or .ds-logs- indices.
-        /usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep close | awk '{print $1}' | grep -v "so-case" | grep -q -E "(logstash-|so-|.ds-logs-)"
-}
-
-# Check for 2 conditions:
-# 1. Are Elasticsearch indices using more disk space than LOG_SIZE_LIMIT?
-# 2. Are there any closed indices that we can delete?
-# If both conditions are true, keep on looping until one of the conditions is false.
-
-while overlimit && closedindices; do
-        CLOSED_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep close | awk '{print $1}' | grep -v "so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3)
-        # We iterate through the closed indices
-	for CLOSED_INDEX in ${CLOSED_INDICES}; do
-	  # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream
-          # To do so, we need to identify to which data stream this index is associated
-          # We extract the data stream name using the pattern below
-          DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+"
-          DATASTREAM=$(echo "${CLOSED_INDEX}" | grep -oE "$DATASTREAM_PATTERN")
-	  # We look up the data stream, and determine the write index
-          CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name)
-	  # We make sure we are not trying to delete a write index
-          if [ "${CLOSED_INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then      
-            # This should not be a write index, so we should be allowed to delete it
-            /usr/sbin/so-elasticsearch-query ${CLOSED_INDEX} -XDELETE
-            # Finally, write a log entry that says we deleted it.
-            echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT} GB) - Index ${CLOSED_INDEX} deleted ..." >> ${LOG}
-          fi
-          if ! overlimit; then
-            exit
-          fi
-        done
-done
-
-while overlimit; do
-
-	# We need to determine the oldest open index.
-	# First, get the list of open indices using _cat/indices?h=index,status | grep open | awk '{print $1}'.
-        # Next, filter out any so-case indices and only select the remaining logstash-, so-, or .ds-logs- indices.
-        # Then, sort by date by telling sort to use hyphen as delimiter and sort on the third field.
-	OPEN_INDICES=$(/usr/sbin/so-elasticsearch-query _cat/indices?h=index,status | grep open | awk '{print $1}' | grep -v "so-case" | grep -E "(logstash-|so-|.ds-logs-)" | sort -t- -k3)
-	# We iterate through the open indices
-        for OPEN_INDEX in ${OPEN_INDICES}; do
-	  # Now that we've sorted the indices from oldest to newest, we need to check each index to see if it is assigned as the current write index for a data stream
-          # To do so, we need to identify to which data stream this index is associated
-          # We extract the data stream name using the pattern below
-          DATASTREAM_PATTERN="logs-[a-zA-Z_.]+-[a-zA-Z_.]+"
-          DATASTREAM=$(echo "${OPEN_INDEX}" | grep -oE "$DATASTREAM_PATTERN")
-	  # We look up the data stream, and determine the write index
-          CURRENT_WRITE_INDEX=$(/usr/sbin/so-elasticsearch-query _data_stream/$DATASTREAM | jq -r .data_streams[0].indices[-1].index_name)
-	  # We make sure we are not trying to delete a write index
-          if [ "${OPEN_INDEX}" != "${CURRENT_WRITE_INDEX}" ]; then      
-            # This should not be a write index, so we should be allowed to delete it
-            /usr/sbin/so-elasticsearch-query ${OPEN_INDEX} -XDELETE
-            # Finally, write a log entry that says we deleted it.
-            echo "$(date) - Used disk space exceeds LOG_SIZE_LIMIT (${LOG_SIZE_LIMIT} GB) - Index ${OPEN_INDEX} deleted ..." >> ${LOG}
-          fi
-          if ! overlimit; then
-            exit
-          fi
-        done
-done

salt/curator/files/curator.yml

@@ -4,9 +4,9 @@
 # Elastic License 2.0.
 
 {% from 'vars/globals.map.jinja' import GLOBALS %}
-{% if grains['role'] in ['so-searchnode', 'so-heavynode'] %}
+{% if GLOBALS.role in ['so-searchnode', 'so-heavynode'] %}
   {%- set elasticsearch = GLOBALS.node_ip -%}
-{% elif grains['role'] in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %}
+{% elif GLOBALS.role in ['so-eval', 'so-managersearch', 'so-standalone', 'so-manager'] %}
   {%- set elasticsearch = GLOBALS.manager_ip -%}
 {%- endif %}
 {%-   set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
@@ -30,10 +30,8 @@ elasticsearch:
       id:
       api_key:
     master_only: False
-{%- if salt['pillar.get']('elasticsearch:auth:enabled') is sameas true %}
     username: "{{ ES_USER }}"
     password: "{{ ES_PASS }}"
-{%- endif %}
 
 logging:
   loglevel: INFO

salt/curator/init.sls

@@ -3,153 +3,11 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-{% from 'allowed_states.map.jinja' import allowed_states %}
+{% from 'curator/map.jinja' import CURATORMERGED %}
-{% if sls in allowed_states %}
-{% from 'vars/globals.map.jinja' import GLOBALS %}
-{% from 'docker/docker.map.jinja' import DOCKER %}
-{% from "curator/map.jinja" import CURATOROPTIONS %}
-{% from "curator/map.jinja" import CURATORMERGED %}
-{% set REMOVECURATORCRON = False %}
-
-# Curator
-# Create the group
-curatorgroup:
-  group.present:
-    - name: curator
-    - gid: 934
-
-# Add user
-curator:
-  user.present:
-    - uid: 934
-    - gid: 934
-    - home: /opt/so/conf/curator
-    - createhome: False
-
-# Create the log directory
-curlogdir:
-  file.directory:
-    - name: /opt/so/log/curator
-    - user: 934
-    - group: 939
-
-curactiondir:
-  file.directory:
-    - name: /opt/so/conf/curator/action
-    - user: 934
-    - group: 939
-    - makedirs: True
-
-actionconfs:
-  file.recurse:
-    - name: /opt/so/conf/curator/action
-    - source: salt://curator/files/action
-    - user: 934
-    - group: 939
-    - template: jinja
-    - defaults:
-        CURATORMERGED: {{ CURATORMERGED }}
-        
-curconf:
-  file.managed:
-    - name: /opt/so/conf/curator/curator.yml
-    - source: salt://curator/files/curator.yml
-    - user: 934
-    - group: 939
-    - mode: 660
-    - template: jinja
-    - show_changes: False
-
-curclusterclose: 
-  file.managed:
-    - name: /usr/sbin/so-curator-cluster-close
-    - source: salt://curator/files/bin/so-curator-cluster-close
-    - user: 934
-    - group: 939
-    - mode: 755
-    - template: jinja
-
-curclusterdelete:
-  file.managed:
-    - name: /usr/sbin/so-curator-cluster-delete
-    - source: salt://curator/files/bin/so-curator-cluster-delete
-    - user: 934
-    - group: 939
-    - mode: 755
-
-curclusterdeletedelete:
-  file.managed:
-    - name: /usr/sbin/so-curator-cluster-delete-delete
-    - source: salt://curator/files/bin/so-curator-cluster-delete-delete
-    - user: 934
-    - group: 939
-    - mode: 755
-    - template: jinja
-
-so-curator:
-  docker_container.{{ CURATOROPTIONS.status }}:
-    - image: {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-curator:{{ GLOBALS.so_version }}
-    - start: {{ CURATOROPTIONS.start }}
-    - hostname: curator
-    - name: so-curator
-    - user: curator
-    - networks:
-      - sobridge:
-        - ipv4_address: {{ DOCKER.containers['so-curator'].ip }}
-    - interactive: True
-    - tty: True
-    - binds:
-      - /opt/so/conf/curator/curator.yml:/etc/curator/config/curator.yml:ro
-      - /opt/so/conf/curator/action/:/etc/curator/action:ro
-      - /opt/so/log/curator:/var/log/curator:rw
-    - require:
-      - file: actionconfs
-      - file: curconf
-      - file: curlogdir
-    - watch:
-      - file: curconf
-
-append_so-curator_so-status.conf:
-  file.append:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - text: so-curator
-    - unless: grep -q so-curator /opt/so/conf/so-status/so-status.conf
-  {% if not CURATOROPTIONS.start %}
-so-curator_so-status.disabled:
-  file.comment:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - regex: ^so-curator$
-  {% else %}
-delete_so-curator_so-status.disabled:
-  file.uncomment:
-    - name: /opt/so/conf/so-status/so-status.conf
-    - regex: ^so-curator$
-  {% endif %}
-
-so-curatorclusterclose:
-  cron.present:
-    - name: /usr/sbin/so-curator-cluster-close > /opt/so/log/curator/cron-close.log 2>&1
-    - user: root
-    - minute: '2'
-    - hour: '*/1'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
-
-so-curatorclusterdeletecron:
-  cron.present:
-    - name: /usr/sbin/so-curator-cluster-delete > /opt/so/log/curator/cron-cluster-delete.log 2>&1
-    - user: root
-    - minute: '*/5'
-    - hour: '*'
-    - daymonth: '*'
-    - month: '*'
-    - dayweek: '*'
 
+include:
+{% if CURATORMERGED.enabled %}
+  - curator.enabled
 {% else %}
-
+  - curator.disabled
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
 {% endif %}

salt/curator/map.jinja

@@ -1,18 +1,7 @@
-{% set CURATOROPTIONS = {} %}
+{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-{% set ENABLED = salt['pillar.get']('curator:enabled', True) %}
+   or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-{% do CURATOROPTIONS.update({'manage_sostatus': True}) %}
+   https://securityonion.net/license; you may not use this file except in compliance with the
-
+   Elastic License 2.0. #}
-# don't start the docker container if curator is disabled via pillar
-{% if not ENABLED %}
-  {% do CURATOROPTIONS.update({'start': False}) %}
-  {% do CURATOROPTIONS.update({'status': 'absent'}) %}
-  {% if (TRUECLUSTER and grains.id.split('_')|last == 'searchnode') or (not TRUECLUSTER and grains.id.split('_')|last == 'manager') %}
-    {% do CURATOROPTIONS.update({'manage_sostatus': False}) %}
-  {% endif %}
-{% else %}
-  {% do CURATOROPTIONS.update({'start': True}) %}
-  {% do CURATOROPTIONS.update({'status': 'running'}) %}
-{% endif %}
 
 {% import_yaml 'curator/defaults.yaml' as CURATORDEFAULTS %}
-{% set CURATORMERGED = salt['pillar.get']('elasticsearch:index_settings', CURATORDEFAULTS.elasticsearch.index_settings, merge=true) %}
+{% set CURATORMERGED = salt['pillar.get']('curator', CURATORDEFAULTS.curator, merge=true) %}

salt/curator/soc_curator.yaml

@@ -0,0 +1,108 @@
+curator:
+  enabled:
+    description: You can enable or disable Curator.
+    helpLink: curator.html
+  elasticsearch:
+    index_settings:
+      logs-import-so:
+        close: &close
+          description: Age, in days, when Curator closes the index.
+          helpLink: curator.html
+          forcedType: int
+        delete: &delete
+          description: Age, in days, when Curator deletes the index.
+          helpLink: curator.html
+          forcedType: int
+      logs-strelka-so:
+        close: *close
+        delete: *delete
+      logs-suricata-so:
+        close: *close
+        delete: *delete
+      logs-syslog-so:
+        close: *close
+        delete: *delete
+      logs-zeek-so:
+        close: *close
+        delete: *delete
+      logs-elastic_agent-metricbeat-default:
+        close: *close
+        delete: *delete
+      logs-elastic_agent-osquerybeat-default:
+        close: *close
+        delete: *delete
+      logs-elastic_agent-fleet_server-default:
+        close: *close
+        delete: *delete
+      logs-elastic_agent-filebeat-default:
+        close: *close
+        delete: *delete
+      logs-elastic_agent-default:
+        close: *close
+        delete: *delete
+      logs-system-auth-default:
+        close: *close
+        delete: *delete
+      logs-system-application-default:
+        close: *close
+        delete: *delete
+      logs-system-security-default:
+        close: *close
+        delete: *delete
+      logs-system-system-default:
+        close: *close
+        delete: *delete
+      logs-system-syslog-default:
+        close: *close
+        delete: *delete
+      logs-windows-powershell-default:
+        close: *close
+        delete: *delete
+      logs-windows-sysmon_operational-default:
+        close: *close
+        delete: *delete
+      so-beats:
+        close: *close
+        delete: *delete
+      so-elasticsearch:
+        close: *close
+        delete: *delete
+      so-firewall:
+        close: *close
+        delete: *delete
+      so-ids:
+        close: *close
+        delete: *delete
+      so-import:
+        close: *close
+        delete: *delete
+      so-kratos:
+        close: *close
+        delete: *delete
+      so-kibana:
+        close: *close
+        delete: *delete
+      so-logstash:
+        close: *close
+        delete: *delete
+      so-netflow:
+        close: *close
+        delete: *delete
+      so-osquery:
+        close: *close
+        delete: *delete
+      so-ossec:
+        close: *close
+        delete: *delete
+      so-redis:
+        close: *close
+        delete: *delete
+      so-strelka:
+        close: *close
+        delete: *delete
+      so-syslog:
+        close: *close
+        delete: *delete
+      so-zeek:
+        close: *close
+        delete: *delete

salt/curator/sostatus.sls

@@ -0,0 +1,21 @@
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+{% from 'allowed_states.map.jinja' import allowed_states %}
+{% if sls.split('.')[0] in allowed_states %}
+
+append_so-curator_so-status.conf:
+  file.append:
+    - name: /opt/so/conf/so-status/so-status.conf
+    - text: so-curator
+    - unless: grep -q so-curator /opt/so/conf/so-status/so-status.conf
+
+{% else %}
+
+{{sls}}_state_not_allowed:
+  test.fail_without_changes:
+    - name: {{sls}}_state_not_allowed
+
+{% endif %}