From a4dc63f3a4e2c9ec4796647fc2afa6a1310d8145 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 14 Sep 2022 09:53:57 -0400
Subject: [PATCH 01/16] Change how zeek and suri are populated in the minion
 file

---
 salt/common/tools/sbin/so-minion          | 16 +++----
 salt/learn/files/logscan.conf             |  7 ---
 salt/learn/init.sls                       | 19 --------
 salt/learn/logscan.sls                    | 58 -----------------------
 salt/suricata/defaults.yaml               | 26 ++++++++--
 salt/zeek/{map.jinja => config.map.jinja} |  0
 salt/zeek/fileextraction_defaults.yaml    | 29 ------------
 salt/zeek/init.sls                        |  2 +-
 8 files changed, 31 insertions(+), 126 deletions(-)
 delete mode 100644 salt/learn/files/logscan.conf
 delete mode 100644 salt/learn/init.sls
 delete mode 100644 salt/learn/logscan.sls
 rename salt/zeek/{map.jinja => config.map.jinja} (100%)
 delete mode 100644 salt/zeek/fileextraction_defaults.yaml

diff --git a/salt/common/tools/sbin/so-minion b/salt/common/tools/sbin/so-minion
index 858d2706c..24544940c 100755
--- a/salt/common/tools/sbin/so-minion
+++ b/salt/common/tools/sbin/so-minion
@@ -160,16 +160,14 @@ function add_patch_pillar_to_minion() {
 function add_sensor_to_minion() {
 	echo "sensor:" >> $PILLARFILE
 	echo "  interface: '$INTERFACE'" >> $PILLARFILE
-	echo "  zeekpin: False" >> $PILLARFILE
-	echo "  zeekpins:" >> $PILLARFILE
-	echo "    - 1" >> $PILLARFILE
-	echo "  zeek_lbprocs: $CORECOUNT" >> $PILLARFILE
-	echo "  suripin: False" >> $PILLARFILE
-	echo "  suripins:" >> $PILLARFILE
-	echo "    - 2" >> $PILLARFILE
-	echo "  suriprocs: $CORECOUNT"  >> $PILLARFILE
 	echo "  mtu: 9000" >> $PILLARFILE
-	echo "  uniqueid: $(date '+%s')" >> $PILLARFILE
+	echo "zeek:" >> $PILLARFILE
+	echo "    config:" >> $PILLARFILE
+	echo "        lb_procs: $CORECOUNT" >> $PILLARFILE
+	echo "suricata:" >> $PILLARFILE
+	echo "    config:" >> $PILLARFILE
+	echo "        af-packet:" >> $PILLARFILE
+	echo "            threads: $CORECOUNT" >> $PILLARFILE
 	echo "steno:" >> $PILLARFILE
 	echo "  stenopin: False" >> $PILLARFILE
 	echo "  stenopins:" >> $PILLARFILE
diff --git a/salt/learn/files/logscan.conf b/salt/learn/files/logscan.conf
deleted file mode 100644
index d7aa30734..000000000
--- a/salt/learn/files/logscan.conf
+++ /dev/null
@@ -1,7 +0,0 @@
-[global]
-ts_format = iso8601
-scan_interval = 30s
-log_level = info
-
-[kratos]
-log_path = kratos/kratos.log
diff --git a/salt/learn/init.sls b/salt/learn/init.sls
deleted file mode 100644
index fb5b89802..000000000
--- a/salt/learn/init.sls
+++ /dev/null
@@ -1,19 +0,0 @@
-{% from 'allowed_states.map.jinja' import allowed_states %}
-{% if sls in allowed_states %}
-
-{% set module_dict = salt['pillar.get']('learn:modules', {} ) %}
-
-{% if module_dict.items()|length != 0 %}
-include:
-{% for module, _ in module_dict.items() %}
-  - 'learn.{{ module }}'
-{% endfor %}
-{% endif %}
-
-{% else %}
-
-{{sls}}_state_not_allowed:
-  test.fail_without_changes:
-    - name: {{sls}}_state_not_allowed
-
-{% endif %}
diff --git a/salt/learn/logscan.sls b/salt/learn/logscan.sls
deleted file mode 100644
index 91f64420a..000000000
--- a/salt/learn/logscan.sls
+++ /dev/null
@@ -1,58 +0,0 @@
-{% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
-{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
-{% set MANAGER = salt['grains.get']('master') %}
-{% set logscan_cpu_period = salt['pillar.get']('learn:modules:logscan:cpu_period', 20000) %}
-{% set enabled = salt['pillar.get']('learn:modules:logscan:enabled', False) %}
-
-{% if enabled %}
-  {% set container_action = 'running' %}
-{% else %}
-  {% set container_action = 'absent'%}
-{% endif %}
-
-
-logscan_data_dir:
-  file.directory:
-    - name: /nsm/logscan/data
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-logscan_conf_dir:
-  file.directory:
-    - name: /opt/so/conf/logscan
-    - user: 939
-    - group: 939
-    - makedirs: True
-
-logscan_conf:
-  file.managed:
-    - name: /opt/so/conf/logscan/logscan.conf
-    - source: salt://learn/files/logscan.conf
-    - user: 939
-    - group: 939
-    - mode: 600
-
-logscan_log_dir:
-  file.directory:
-    - name: /opt/so/log/logscan
-    - user: 939
-    - group: 939
-
-so-logscan:
-  docker_container.{{ container_action }}:
-    {% if container_action == 'running' %}
-    - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-logscan:{{ VERSION }}
-    - hostname: logscan
-    - name: so-logscan
-    - binds:
-      - /nsm/logscan/data:/logscan/data:rw
-      - /opt/so/conf/logscan/logscan.conf:/logscan/logscan.conf:ro
-      - /opt/so/log/logscan:/logscan/output:rw
-      - /opt/so/log:/logscan/logs:ro
-    - cpu_period: {{ logscan_cpu_period }}
-    - require:
-      - file: logscan_conf
-    {% else %}
-    - force: true
-    {% endif %}
diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml
index 0fe3b444f..4e4d28a87 100644
--- a/salt/suricata/defaults.yaml
+++ b/salt/suricata/defaults.yaml
@@ -1,5 +1,28 @@
 suricata:
   config:
+    threading:
+        set-cpu-affinity: 'no'
+        detect-thread-ratio: 1.0
+        cpu-affinity:
+            - management-cpu-set:
+                cpu: []
+            - receive-cpu-set:
+                cpu: []
+            - worker-cpu-set:
+                cpu: []
+                mode: exclusive
+                threads: 1
+                prio:
+                    default: high
+    af-packet:
+        interface: bond0
+        cluster-id: 59
+        cluster-type: cluster_flow
+        defrag: true
+        use-mmap: true
+        threads: 1
+        tpacket-v3: true
+        ring-size: 5000
     vars:
       address-groups:
         HOME_NET: "[192.168.0.0/16,10.0.0.0/8,172.16.0.0/12]"
@@ -348,9 +371,6 @@ suricata:
           include-mpm-stats: false
     mpm-algo: auto
     spm-algo: auto
-    threading:
-      set-cpu-affinity: "yes"
-      detect-thread-ratio: 1.0
     luajit:
       states: 128
 
diff --git a/salt/zeek/map.jinja b/salt/zeek/config.map.jinja
similarity index 100%
rename from salt/zeek/map.jinja
rename to salt/zeek/config.map.jinja
diff --git a/salt/zeek/fileextraction_defaults.yaml b/salt/zeek/fileextraction_defaults.yaml
deleted file mode 100644
index 3823b8203..000000000
--- a/salt/zeek/fileextraction_defaults.yaml
+++ /dev/null
@@ -1,29 +0,0 @@
-zeek:
-  policy:
-    file_extraction:
-      - application/x-dosexec: exe
-      - application/pdf: pdf
-      - application/msword: doc
-      - application/vnd.ms-powerpoint: doc
-      - application/rtf: doc
-      - application/vnd.ms-word.document.macroenabled.12: doc
-      - application/vnd.ms-word.template.macroenabled.12: doc
-      - application/vnd.ms-powerpoint.template.macroenabled.12: doc
-      - application/vnd.ms-excel: doc
-      - application/vnd.ms-excel.addin.macroenabled.12: doc
-      - application/vnd.ms-excel.sheet.binary.macroenabled.12: doc
-      - application/vnd.ms-excel.template.macroenabled.12: doc
-      - application/vnd.ms-excel.sheet.macroenabled.12: doc
-      - application/vnd.openxmlformats-officedocument.presentationml.presentation: doc
-      - application/vnd.openxmlformats-officedocument.presentationml.slide: doc
-      - application/vnd.openxmlformats-officedocument.presentationml.slideshow: doc
-      - application/vnd.openxmlformats-officedocument.presentationml.template: doc
-      - application/vnd.openxmlformats-officedocument.spreadsheetml.sheet: doc
-      - application/vnd.openxmlformats-officedocument.spreadsheetml.template: doc
-      - application/vnd.openxmlformats-officedocument.wordprocessingml.document: doc
-      - application/vnd.openxmlformats-officedocument.wordprocessingml.template: doc
-      - application/vnd.ms-powerpoint.addin.macroenabled.12: doc
-      - application/vnd.ms-powerpoint.slide.macroenabled.12: doc
-      - application/vnd.ms-powerpoint.presentation.macroenabled.12: doc
-      - application/vnd.ms-powerpoint.slideshow.macroenabled.12: doc
-      - application/vnd.openxmlformats-officedocument: doc
\ No newline at end of file
diff --git a/salt/zeek/init.sls b/salt/zeek/init.sls
index 6185308ac..b6f3231ae 100644
--- a/salt/zeek/init.sls
+++ b/salt/zeek/init.sls
@@ -6,7 +6,7 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
 
-{% from "zeek/map.jinja" import ZEEKOPTIONS with context %}
+{% from "zeek/config.map.jinja" import ZEEKOPTIONS with context %}
 
 {% set VERSION = salt['pillar.get']('global:soversion') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}

From 0a40bfcb887f21a9cd7c57e2d3864ed87a9102d6 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 14 Sep 2022 11:00:22 -0400
Subject: [PATCH 02/16] Change how pcap is written to the minion file

---
 salt/common/tools/sbin/so-minion |  7 -------
 salt/pcap/defaults.yaml          | 11 +++++++++++
 salt/pcap/soc_pcap.yaml          |  8 ++++----
 3 files changed, 15 insertions(+), 11 deletions(-)
 create mode 100644 salt/pcap/defaults.yaml

diff --git a/salt/common/tools/sbin/so-minion b/salt/common/tools/sbin/so-minion
index 24544940c..58f04ab77 100755
--- a/salt/common/tools/sbin/so-minion
+++ b/salt/common/tools/sbin/so-minion
@@ -168,13 +168,6 @@ function add_sensor_to_minion() {
 	echo "    config:" >> $PILLARFILE
 	echo "        af-packet:" >> $PILLARFILE
 	echo "            threads: $CORECOUNT" >> $PILLARFILE
-	echo "steno:" >> $PILLARFILE
-	echo "  stenopin: False" >> $PILLARFILE
-	echo "  stenopins:" >> $PILLARFILE
-	echo "    - 3" >> $PILLARFILE
-	echo "  enabled: True" >> $PILLARFILE
-	echo "  disks:" >> $PILLARFILE
-	echo "    - '/some/path'" >> $PILLARFILE
 }
 
 function createSTANDALONE() {
diff --git a/salt/pcap/defaults.yaml b/salt/pcap/defaults.yaml
new file mode 100644
index 000000000..bb93ec6cc
--- /dev/null
+++ b/salt/pcap/defaults.yaml
@@ -0,0 +1,11 @@
+pcap:
+    enabled: True
+    config:
+        maxdirectoryfiles: 30000
+        diskfreepercentage: 10
+        blocks: 2048
+        preallocate_file_mb: 4096
+        aiops: 128
+        stenopin: False
+        stenopins: []
+        disks: []
\ No newline at end of file
diff --git a/salt/pcap/soc_pcap.yaml b/salt/pcap/soc_pcap.yaml
index 515dd346b..68eb0f083 100644
--- a/salt/pcap/soc_pcap.yaml
+++ b/salt/pcap/soc_pcap.yaml
@@ -1,9 +1,9 @@
 pcap:
+  enabled: 
+  description: Enable or Disable Stenographer on all sensors or a single sensor
   config:
-    enabled: 
-      description: Enable or Disable Stenographer on all sensors or a single sensor
-    maxfiles:
-      description: The maximum number of packet/index files to create before cleaning old ones up. 
+    maxdirectoryfiles:
+      description: The maximum number of packet/index files to create before deleting old files. The default is about 8 days regardless of free space. 
     diskfreepercentage:
       description: The disk space percent to always keep free for pcap
     blocks:

From be4c15877aab996ba5e32aa2244eaeda8caeec41 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 14 Sep 2022 11:11:21 -0400
Subject: [PATCH 03/16] Improve pcap defaults

---
 salt/pcap/soc_pcap.yaml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/salt/pcap/soc_pcap.yaml b/salt/pcap/soc_pcap.yaml
index 68eb0f083..053874da6 100644
--- a/salt/pcap/soc_pcap.yaml
+++ b/salt/pcap/soc_pcap.yaml
@@ -8,5 +8,17 @@ pcap:
       description: The disk space percent to always keep free for pcap
     blocks:
       description: The number of 1MB packet blocks used by AF_PACKET to store packets in memory, per thread. You shouldn't need to change this. 
+      advanced: True
     preallocate_file_mb:
       description: File size to pre-allocate for individual pcap files. You shouldn't need to change this. 
+      advanced: True
+    aiops:
+      description: The max number of async writes to allow at once.
+      advanced: True
+    pin_to_cpu:
+      description: Enable CPU pinning for PCAP.
+    cpus_to_pin_to:
+      description: CPU to pin PCAP to. Currently only a single SPU is supported
+    disks:
+      description: List of disks to use for PCAP. This is currently not used.
+      advanced: True

From 547abb0fe1ae23d44dddd2957bcec779c2ad8dfa Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 14 Sep 2022 12:12:33 -0400
Subject: [PATCH 04/16] Remove learn

---
 salt/allowed_states.map.jinja   |   15 +-
 salt/common/tools/sbin/so-learn |  295 --------
 salt/top.sls                    |    5 -
 setup/so-setup.old              | 1146 -------------------------------
 4 files changed, 5 insertions(+), 1456 deletions(-)
 delete mode 100755 salt/common/tools/sbin/so-learn
 delete mode 100755 setup/so-setup.old

diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja
index ed530ac91..fa74b3751 100644
--- a/salt/allowed_states.map.jinja
+++ b/salt/allowed_states.map.jinja
@@ -47,8 +47,7 @@
           'schedule',
           'soctopus',
           'tcpreplay',
-          'docker_clean',
-          'learn'
+          'docker_clean'
           ],
       'so-heavynode': [
           'ssl',
@@ -106,8 +105,7 @@
           'zeek',
           'schedule',
           'tcpreplay',
-          'docker_clean',
-          'learn'
+          'docker_clean'
           ],
       'so-manager': [
           'salt.master',
@@ -128,8 +126,7 @@
           'utility',
           'schedule',
           'soctopus',
-          'docker_clean',
-          'learn'
+          'docker_clean'
           ],
       'so-managersearch': [
           'salt.master',
@@ -150,8 +147,7 @@
           'utility',
           'schedule',
           'soctopus',
-          'docker_clean',
-          'learn'
+          'docker_clean'
           ],
       'so-searchnode': [
           'ssl',
@@ -184,8 +180,7 @@
           'schedule',
           'soctopus',
           'tcpreplay',
-          'docker_clean',
-          'learn'
+          'docker_clean'
           ],
       'so-sensor': [
           'ssl',
diff --git a/salt/common/tools/sbin/so-learn b/salt/common/tools/sbin/so-learn
deleted file mode 100755
index 2b766c738..000000000
--- a/salt/common/tools/sbin/so-learn
+++ /dev/null
@@ -1,295 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-from itertools import chain
-from typing import List
-
-import signal
-import sys
-import os
-import re
-import subprocess
-import argparse
-import textwrap
-import yaml
-import multiprocessing
-import docker
-import pty
-
-minion_pillar_dir = '/opt/so/saltstack/local/pillar/minions'
-so_status_conf = '/opt/so/conf/so-status/so-status.conf'
-proc: subprocess.CompletedProcess = None
-
-# Temp store of modules, will likely be broken out into salt
-def get_learn_modules():
-  return {
-    'logscan': { 'cpu_period': get_cpu_period(fraction=0.25), 'enabled': False, 'description': 'Scan log files against pre-trained models to alert on anomalies.' }
-  }
-
-
-def get_cpu_period(fraction: float):
-  multiplier = 10000
-  
-  num_cores = multiprocessing.cpu_count()
-  if num_cores <= 2:
-    fraction = 1.
-  
-  num_used_cores = int(num_cores * fraction)
-  cpu_period = num_used_cores * multiplier
-  return cpu_period
-
-
-def sigint_handler(*_):
-  print('Exiting gracefully on Ctrl-C')
-  if proc is not None: proc.send_signal(signal.SIGINT)
-  sys.exit(1)
-
-
-def find_minion_pillar() -> str:
-  regex = '^.*_(manager|managersearch|standalone|import|eval)\.sls$'
-  
-  result = []
-  for root, _, files in os.walk(minion_pillar_dir):
-    for f_minion_id in files:
-      if re.search(regex, f_minion_id):
-        result.append(os.path.join(root, f_minion_id))
-  
-  if len(result) == 0:
-    print('Could not find manager-type pillar (eval, standalone, manager, managersearch, import). Are you running this script on the manager?', file=sys.stderr)
-    sys.exit(3)
-  elif len(result) > 1:
-    res_str = ', '.join(f'\"{result}\"')
-    print('(This should not happen, the system is in an error state if you see this message.)\n', file=sys.stderr)
-    print('More than one manager-type pillar exists, minion id\'s listed below:', file=sys.stderr)
-    print(f'  {res_str}', file=sys.stderr)
-    sys.exit(3)
-  else:
-    return result[0]
-
-
-def read_pillar(pillar: str):
-  try:
-    with open(pillar, 'r') as pillar_file:
-      loaded_yaml = yaml.safe_load(pillar_file.read())
-      if loaded_yaml is None:
-        print(f'Could not parse {pillar}', file=sys.stderr)
-        sys.exit(3)
-      return loaded_yaml
-  except:
-    print(f'Could not open {pillar}', file=sys.stderr)
-    sys.exit(3)
-
-
-def write_pillar(pillar: str, content: dict):
-  try:
-    with open(pillar, 'w') as pillar_file:
-      yaml.dump(content, pillar_file, default_flow_style=False)
-  except:
-    print(f'Could not open {pillar}', file=sys.stderr)
-    sys.exit(3)
-
-
-def mod_so_status(action: str, item: str):
-  with open(so_status_conf, 'a+') as conf:
-    conf.seek(0)
-    containers = conf.readlines()
-    
-    if f'so-{item}\n' in containers:
-      if action == 'remove': containers.remove(f'so-{item}\n')
-      if action == 'add': pass
-    else:
-      if action == 'remove': pass
-      if action == 'add': containers.append(f'so-{item}\n')
-      
-    [containers.remove(c_name) for c_name in containers if c_name == '\n'] # remove extra newlines
-    
-    conf.seek(0)
-    conf.truncate(0)
-    conf.writelines(containers)
-
-
-def create_pillar_if_not_exist(pillar:str, content: dict):
-  pillar_dict = content
-
-  if pillar_dict.get('learn', {}).get('modules') is None:
-    pillar_dict['learn'] = {}
-    pillar_dict['learn']['modules'] = get_learn_modules()
-    content.update()
-    write_pillar(pillar, content)
-  
-  return content
-
-
-def salt_call(module: str):
-  salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', f'learn.{module}', 'queue=True']
-
-  print(f'  Applying salt state for {module} module...')
-  proc = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-  return_code = proc.returncode
-  if return_code != 0:
-    print(f'  [ERROR] Failed to apply salt state for {module} module.')
-
-  return return_code
-
-
-def pull_image(module: str):
-  container_basename = f'so-{module}'
-
-  client = docker.from_env()
-  image_list = client.images.list(filters={ 'dangling': False })  
-  tag_list = list(chain.from_iterable(list(map(lambda x: x.attrs.get('RepoTags'), image_list))))
-  basename_match = list(filter(lambda x: f'{container_basename}' in x, tag_list))
-  local_registry_match = list(filter(lambda x: ':5000' in x, basename_match))
-
-  if len(local_registry_match) == 0:
-    print(f'Pulling and verifying missing image for {module} (may take several minutes) ...')
-    pull_command = ['so-image-pull', '--quiet', container_basename]
-  
-    proc = subprocess.run(pull_command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-    return_code = proc.returncode
-    if return_code != 0:
-      print(f'[ERROR] Failed to pull image so-{module}, skipping state.')
-  else:
-    return_code = 0
-  return return_code
-
-
-def apply(module_list: List):
-  return_code = 0
-  for module in module_list:
-    salt_ret = salt_call(module)
-    # Only update return_code if the command returned a non-zero return
-    if salt_ret != 0:
-      return_code = salt_ret
-    
-  return return_code
-
-
-def check_apply(args: dict):
-  if args.apply:
-    print('Configuration updated. Applying changes:')
-    return apply(args.modules)
-  else:
-    message = 'Configuration updated. Would you like to apply your changes now? (y/N) '
-    answer = input(message)
-    while answer.lower() not in [ 'y', 'n', '' ]:
-      answer = input(message)
-    if answer.lower() in [ 'n', '' ]:
-      return 0
-    else:
-      print('Applying changes:')
-      return apply(args.modules)
-
-
-def enable_disable_modules(args, enable: bool):
-  pillar_modules = args.pillar_dict.get('learn', {}).get('modules')
-  pillar_mod_names = args.pillar_dict.get('learn', {}).get('modules').keys()
-  
-  action_str = 'add' if enable else 'remove'
-
-  if 'all' in args.modules:
-    for module, details in pillar_modules.items():
-      details['enabled'] = enable
-      mod_so_status(action_str, module)
-      if enable: pull_image(module)
-    args.pillar_dict.update()
-    write_pillar(args.pillar, args.pillar_dict)
-  else:
-    write_needed = False
-    for module in args.modules:
-      if module in pillar_mod_names:
-        if pillar_modules[module]['enabled'] == enable:
-          state_str = 'enabled' if enable else 'disabled'
-          print(f'{module} module already {state_str}.', file=sys.stderr)
-        else:
-          if enable and pull_image(module) != 0:
-            continue
-          pillar_modules[module]['enabled'] = enable
-          mod_so_status(action_str, module)
-          write_needed = True
-    if write_needed:
-      args.pillar_dict.update()
-      write_pillar(args.pillar, args.pillar_dict)
-  
-  cmd_ret = check_apply(args)
-  return cmd_ret
-
-
-def enable_modules(args):
-  enable_disable_modules(args, enable=True)
-
-
-def disable_modules(args):
-  enable_disable_modules(args, enable=False)
-
-
-def list_modules(*_):
-  print('Available ML modules:')
-  for module, details in get_learn_modules().items():
-    print(f'  - { module } : {details["description"]}')
-  return 0
-
-
-def main():
-  beta_str = 'BETA - SUBJECT TO CHANGE\n'
-
-  apply_help='After ACTION the chosen modules, apply any necessary salt states.'
-  enable_apply_help = apply_help.replace('ACTION', 'enabling')
-  disable_apply_help = apply_help.replace('ACTION', 'disabling')
-
-  signal.signal(signal.SIGINT, sigint_handler)
-
-  if os.geteuid() != 0:
-    print('You must run this script as root', file=sys.stderr)
-    sys.exit(1)
-
-  main_parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter)
-
-  subcommand_desc = textwrap.dedent(
-    """\
-      enable              Enable one or more ML modules.
-      disable             Disable one or more ML modules.
-      list                List all available ML modules.
-    """
-  )
-
-  subparsers = main_parser.add_subparsers(title='commands', description=subcommand_desc, metavar='', dest='command')
-
-  module_help_str = 'One or more ML modules, which can be listed using \'so-learn list\'. Use the keyword \'all\' to apply the action to all available modules.'
-
-  enable = subparsers.add_parser('enable')
-  enable.set_defaults(func=enable_modules)
-  enable.add_argument('modules', metavar='ML_MODULE', nargs='+', help=module_help_str)
-  enable.add_argument('--apply', action='store_const', const=True, required=False, help=enable_apply_help)
-
-  disable = subparsers.add_parser('disable')
-  disable.set_defaults(func=disable_modules)
-  disable.add_argument('modules', metavar='ML_MODULE', nargs='+', help=module_help_str)
-  disable.add_argument('--apply', action='store_const', const=True, required=False, help=disable_apply_help)
-
-  list = subparsers.add_parser('list')
-  list.set_defaults(func=list_modules)
-
-  args = main_parser.parse_args(sys.argv[1:])
-  args.pillar = find_minion_pillar()
-  args.pillar_dict = create_pillar_if_not_exist(args.pillar, read_pillar(args.pillar))
-
-  if hasattr(args, 'func'):
-    exit_code = args.func(args)
-  else:
-    if args.command is None:
-      print(beta_str)
-      main_parser.print_help()
-    sys.exit(0)
-
-  sys.exit(exit_code)
-
-
-if __name__ == '__main__':
-  main()
diff --git a/salt/top.sls b/salt/top.sls
index 6dc1f7dc2..e12a3a22e 100644
--- a/salt/top.sls
+++ b/salt/top.sls
@@ -114,7 +114,6 @@ base:
     {%- endif %}
     - docker_clean
     - pipeline.load
-    - learn
 
   '*_manager and G@saltversion:{{saltversion}}':
     - match: compound
@@ -160,7 +159,6 @@ base:
     - playbook
     - docker_clean
     - pipeline.load
-    - learn
 
   '*_standalone and G@saltversion:{{saltversion}}':
     - match: compound
@@ -215,7 +213,6 @@ base:
     - docker_clean
     - elastic-fleet
     - pipeline.load
-    - learn
 
   '*_searchnode and G@saltversion:{{saltversion}}':
     - match: compound
@@ -281,7 +278,6 @@ base:
     - playbook
     - docker_clean
     - pipeline.load
-    - learn
 
   '*_heavynode and G@saltversion:{{saltversion}}':
     - match: compound
@@ -348,7 +344,6 @@ base:
     - schedule
     - docker_clean
     - pipeline.load
-    - learn
 
   '*_receiver and G@saltversion:{{saltversion}}':
     - match: compound
diff --git a/setup/so-setup.old b/setup/so-setup.old
deleted file mode 100755
index d916777fd..000000000
--- a/setup/so-setup.old
+++ /dev/null
@@ -1,1146 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-# Make sure you are root before doing anything
-uid="$(id -u)"
-if [ "$uid" -ne 0 ]; then
-	echo "This script must be run using sudo!"
-	exit 1
-fi
-
-# Save the original argument array since we modify it
-original_args=("$@")
-
-cd "$(dirname "$0")" || exit 255
-
-echo "Getting started..."
-
-# Source the generic function libraries that are also used by the product after
-# setup. These functions are intended to be reusable outside of the setup process.
-source ../salt/common/tools/sbin/so-common
-source ../salt/common/tools/sbin/so-image-common
-
-# Setup bash functionality is divided into functions and user-facing prompts. 
-# Do not attempt to re-use any of this functionality outside of setup. Instead, 
-# if needed, migrated generic functions into so-common.
-source ./so-functions
-source ./so-whiptail
-
-# Finally, source the default variable definitions, which require availability of
-# functions sourced above.
-source ./so-variables
-
-# Parse command line arguments
-setup_type=$1
-automation=$2
-WHATWOULDYOUSAYYAHDOHERE=setup
-
-while [[ $# -gt 0 ]]; do
-	arg="$1"
-	shift
-	case "$arg" in
-		"--turbo="* ) 
-			export TURBO="http://${arg#*=}";;
-		"--proxy="* ) 
-			export {http,https,ftp,rsync,all}_proxy="${arg#*=}";;
-		"--allow-role="* )
-			export ALLOW_ROLE="${arg#*=}";;
-		"--allow-cidr="* )
-			export ALLOW_CIDR="${arg#*=}";;
-		"--skip-reboot" )
-			export SKIP_REBOOT=1;;
-		* )
-			if [[ "$arg" == "--"* ]]; then
-				echo "Invalid option"
-			fi
-	esac
-done
-
-detect_os
-is_analyst=
-if [ "$setup_type" = 'analyst' ]; then
-    is_analyst=true
-    # Check to see if this is an ISO
-    if [ -d /root/SecurityOnion ]; then
-        is_analyst_iso=true
-    fi
-fi
-
-if [[ "$setup_type" == 'iso' ]]; then
-	if [[ $is_centos ]]; then
-		is_iso=true
-	else
-		echo "Only use 'so-setup iso' for an ISO install on CentOS. Please run 'so-setup network' instead."
-		exit 1
-	fi
-fi
-
-# Check to see if this is an analyst install. If it is let's run things differently
-
-if [[ $is_analyst ]]; then
-
-	# Make sure it's CentOS
-	if [[ ! $is_centos ]]; then
-		echo "Analyst Workstation is only supported on CentOS 7"
-		exit 1
-	fi
-
-	if ! whiptail_analyst_install; then
-		if [[ $is_analyst_iso ]]; then
-			if whiptail_analyst_nongrid_iso; then
-				# Remove setup from auto launching
-				parse_install_username
-				sed -i '$ d' /home/$INSTALLUSERNAME/.bash_profile >> "$setup_log" 2>&1
-				echo "Enabling graphical interface and setting it to load at boot"
-				systemctl set-default graphical.target
-				startx
-				exit 0
-			else
-				# Abort!
-				exit 0
-			fi
-		else
-			if whiptail_analyst_nongrid_network; then
-				echo ""
-				echo ""
-				echo "Kicking off the automated setup of the analyst workstation. This can take a while depending on your network connection."
-				echo ""
-				echo ""
-				analyst_salt_local
-			else
-				# Abort!
-				exit 0
-			fi
-		fi
-	fi
-
-	# If you got this far then you want to join the grid
-	is_minion=true
-
-fi
-    
-
-
-if ! [ -f $install_opt_file ] && [ -d /root/manager_setup/securityonion ] && [[ $(pwd) != /root/manager_setup/securityonion/setup ]]; then
-	exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}"
-fi
-
-if [[ -f /root/accept_changes ]]; then
-	is_reinstall=true
-
-	# Move last setup log to backup
-	mv "$setup_log" "$setup_log.bak"
-	[ -f "$error_log" ] && mv "$error_log" "$error_log.bak"
-fi
-
-parse_install_username
-
-if ! [ -f $install_opt_file ]; then
-	# Begin Installation pre-processing
-	title "Initializing Setup"
-	info "Installing as the $INSTALLUSERNAME user"
-
-	analyze_system
-fi
-
-# Set up handler for setup to exit early (use `kill -SIGUSR1 "$setup_proc"; exit 1` in child scripts)
-trap 'catch $LINENO' SIGUSR1
-setup_proc="$$"
-catch() {
-	info "Fatal error occurred at $1 in so-setup, failing setup."
-	grep --color=never "ERROR" "$setup_log" > "$error_log"
-	whiptail_setup_failed
-	exit 1
-}
-automated=no
-progress() {
-	local msg=${1:-'Please wait while installing...'}
-
-	if [ $automated == no ]; then
-		whiptail --title "$whiptail_title" --gauge "$msg" 6 70 0 # append to text
-	else
-		cat >> $setup_log 2>&1
-	fi		
-}
-
-if [[ -f automation/$automation && $(basename $automation) == $automation ]]; then
-	echo "Preselecting variable values based on automated setup: $automation" >> $setup_log 2>&1
-	source automation/$automation
-	automated=yes
-
-	attempt=1
-	attempts=60
-	ip a | grep "$MNIC:" | grep "state UP" >> $setup_log 2>&1
-	while [ $? -ne 0 ]; do
-		ip a >> $setup_log 2>&1
-		if [ $attempt -gt $attempts ]; then
-			echo "Network unavailable - setup cannot continue" >> $setup_log 2>&1
-			exit 1
-		fi
-		echo "Waiting for network to come up (attempt $attempt of $attempts)" >> $setup_log 2>&1
-		attempt=$((attempt + 1))
-		sleep 10;
-		ip a | grep "$MNIC:" | grep "state UP" >> $setup_log 2>&1
-	done
-	echo "Network is up on $MNIC" >> $setup_log 2>&1
-
-	if [[ ! $is_iso ]]; then
-		echo "Installing sshpass for automated testing." >> $setup_log 2>&1
-		if [ "$OS" == ubuntu ]; then
-			retry 50 10 "apt-get -y install sshpass" >> $setup_log 2>&1 || exit 1
-		else
-			yum -y install sshpass >> $setup_log 2>&1
-		fi
-	fi
-fi
-
-case "$setup_type" in
-	iso | network | analyst) # Accepted values
-		echo "Beginning Security Onion $setup_type install" >> $setup_log 2>&1
-		;;
-	*)
-		echo "Invalid install type, must be 'iso', 'network' or 'analyst'." | tee -a $setup_log
-		exit 1
-		;;
-esac
-
-#set ssh commands that will be used based on if this is an automated test install or not
-set_ssh_cmds $automated
-
-# Allow execution of SO tools during setup
-local_sbin="$(pwd)/../salt/common/tools/sbin"
-export PATH=$PATH:$local_sbin
-
-set_palette >> $setup_log 2>&1
-
-# Kernel messages can overwrite whiptail screen #812
-# https://github.com/Security-Onion-Solutions/securityonion/issues/812
-dmesg -D
-
-# Kernel consoleblank is causing whiptail progress screen to appear to hang #1084
-# https://github.com/Security-Onion-Solutions/securityonion/issues/1084
-if [ "$automated" == no ]; then
-	TTY=$(tty)
-	echo "Setup is running on TTY $TTY" >> $setup_log 2>&1
-	if echo $TTY | grep -q "/dev/tty"; then 
-		CONSOLEBLANK=$(cat /sys/module/kernel/parameters/consoleblank)
-		echo "Kernel consoleblank value before: $CONSOLEBLANK" >> $setup_log 2>&1
-		if [ $CONSOLEBLANK -gt 0 ]; then 
-			echo "Running 'setterm -blank 0' for TTY $TTY" >> $setup_log 2>&1
-			TERM=linux setterm -blank 0 >$TTY <$TTY
-			CONSOLEBLANK=$(cat /sys/module/kernel/parameters/consoleblank)
-			echo "Kernel consoleblank value after: $CONSOLEBLANK" >> $setup_log 2>&1
-		fi
-	fi
-fi
-
-if ! [[ -f $install_opt_file ]]; then
-	if (whiptail_you_sure); then
-		true
-	else
-		echo "User cancelled setup." | tee -a "$setup_log"
-		whiptail_cancel
-	fi
-	if [[ $is_analyst ]]; then
-		collect_hostname
-		if [[ $is_analyst_iso ]]; then
-			# Prompt Network Setup
-			whiptail_management_nic
-			whiptail_dhcp_or_static
-			
-			if [ "$address_type" != 'DHCP' ]; then
-				collect_int_ip_mask
-				collect_gateway
-				collect_dns
-				collect_dns_domain
-			fi
-		
-		fi
-		if [[ ! $is_analyst_iso ]]; then
-			# This should be a network install
-			whiptail_network_notice
-			whiptail_dhcp_warn
-			whiptail_management_nic
-		fi
-		whiptail_network_init_notice
-		network_init
-		printf '%s\n' \
-			"MNIC=$MNIC" \
-			"HOSTNAME=$HOSTNAME" > "$net_init_file"
-		set_main_ip
-		compare_main_nic_ip
-
-	fi
-
-	if [[ $setup_type == 'iso' ]] && [ "$automated" == no ]; then
-		whiptail_first_menu_iso
-		if [[ $option == "CONFIGURENETWORK" ]]; then
-			collect_hostname
-			network_init_whiptail
-			whiptail_network_init_notice
-			network_init
-			printf '%s\n' \
-				"MNIC=$MNIC" \
-				"HOSTNAME=$HOSTNAME" > "$net_init_file"
-			set_main_ip
-			compare_main_nic_ip
-			whiptail_net_setup_complete
-		else
-			true
-		fi
-	fi
-	if [[ ! $is_analyst ]]; then
-		whiptail_install_type
-	fi
-else
-	source $install_opt_file
-fi
-
-if [ "$install_type" = 'EVAL' ]; then
-	is_node=true
-	is_manager=true
-	is_sensor=true
-	is_eval=true
-	STRELKARULES=1
-elif [ "$install_type" = 'STANDALONE' ]; then
-	is_manager=true
-	is_distmanager=true
-	is_node=true
-	is_sensor=true
-elif [ "$install_type" = 'MANAGERSEARCH' ]; then
-	is_manager=true
-	is_distmanager=true
-	is_node=true
-elif [ "$install_type" = 'MANAGER' ]; then
-	is_manager=true
-	is_distmanager=true
-elif [ "$install_type" = 'SENSOR' ]; then
-	is_sensor=true
-	is_minion=true
-elif [[ "$install_type" =~ ^('SEARCHNODE'|'HOTNODE'|'WARMNODE')$ ]]; then
-	is_node=true
-	is_minion=true
-elif [ "$install_type" = 'HEAVYNODE' ]; then
-	is_node=true
-	is_minion=true
-	is_sensor=true
-elif [ "$install_type" = 'FLEET' ]; then
-	is_minion=true
-	is_fleet_standalone=true
-	OSQUERY=1
-elif [ "$install_type" = 'IDH' ]; then
-	is_minion=true
-	is_idh=true
-	IDH=1
-elif [ "$install_type" = 'HELIXSENSOR' ]; then
-	is_helix=true
-elif [ "$install_type" = 'IMPORT' ]; then
-	is_import=true
-elif [ "$install_type" = 'RECEIVER' ]; then
-	is_minion=true
-	is_receiver=true
-elif [ "$install_type" = 'ANALYST' ]; then
-	if [ "$setup_type" != 'analyst' ]; then
-		exec bash so-setup analyst
-	fi
-fi
-
-if [[ $is_manager || $is_import ]]; then
-	check_elastic_license
-fi
-
-if ! [[ -f $install_opt_file ]]; then
-	if [[ $is_manager && $is_sensor ]]; then
-		check_requirements "standalone"
-	elif [[ $is_fleet_standalone ]]; then
-		check_requirements "dist" "fleet"
-	elif [[ $is_idh ]]; then
-		check_requirements "dist" "idh"
-	elif [[ $is_sensor && ! $is_eval ]]; then
-		check_requirements "dist" "sensor"
-	elif [[ $is_distmanager || $is_minion ]] && [[ ! ( $is_import || $is_analyst ) ]]; then
-		check_requirements "dist"
-	elif [[ $is_import ]]; then
-		check_requirements "import"
-	fi
-
-	[[ -f $net_init_file ]] && whiptail_net_reinit && reinit_networking=true
-
-	if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then
-		collect_hostname
-	fi
-
-	[[ ! ( $is_eval || $is_import ) ]] && whiptail_node_description
-
-	if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then
-		network_init_whiptail
-	else
-		source "$net_init_file"
-	fi
-
-	if [[ $reinit_networking ]] || ! [[ -f $net_init_file ]]; then
-		whiptail_network_init_notice
-		network_init
-	fi
-
-	set_main_ip
-	compare_main_nic_ip
-
-	if [[ $is_minion ]]; then
-		collect_mngr_hostname
-		add_mngr_ip_to_hosts
-		whiptail_ssh_key_copy_notice
-		copy_ssh_key >> $setup_log 2>&1
-	fi
-
-	if [[ $is_idh ]]; then
-	  collect_idh_services
-	  collect_idh_preferences
-    fi
-
-	# Check if this is an airgap install
-	if [[ ( $is_manager || $is_import) && $is_iso ]]; then
-		whiptail_airgap
-		if [[ "$INTERWEBS" == 'AIRGAP' ]]; then
-			is_airgap=true
-		fi
-	elif [[ $is_minion && ( $is_iso || $is_analyst ) ]]; then
-		$sshcmd -i /root/.ssh/so.key soremote@"$MSRV" [[ -f /etc/yum.repos.d/airgap_repo.repo ]] >> $setup_log 2>&1
-		airgap_check=$?
-		[[ $airgap_check == 0 ]] && is_airgap=true >> $setup_log 2>&1
-	fi
-
-	reset_proxy
-	if [[ -z $is_airgap ]]; then
-		collect_net_method
-		[[ -n "$so_proxy" ]] && set_proxy >> $setup_log 2>&1
-	fi
-
-	if [[ $is_minion ]] && ! (compare_versions); then
-		info "Installer version mismatch, downloading correct version from manager"
-		printf '%s\n' \
-			"install_type=$install_type" \
-			"MNIC=$MNIC" \
-			"HOSTNAME=$HOSTNAME" \
-			"MSRV=$MSRV" \
-			"MSRVIP=$MSRVIP" \
-			"is_airgap=$is_airgap" \
-			"NODE_DESCRIPTION=\"$NODE_DESCRIPTION\"" > "$install_opt_file"
-		[[ -n $so_proxy ]] && echo "so_proxy=$so_proxy" >> "$install_opt_file"
-		download_repo_tarball
-		exec bash /root/manager_setup/securityonion/setup/so-setup "${original_args[@]}"
-	fi
-else
-	rm -rf $install_opt_file >> "$setup_log" 2>&1
-fi
-
-if [[ -z $is_airgap ]]; then
-	percentage=0
-	{
-		installer_progress_loop 'Running preflight checks...' &
-		progress_bg_proc=$!
-		./so-preflight true "$setup_log" >> $setup_log 2>&1
-		preflight_ret=$?
-		echo "$preflight_ret" > /tmp/preflight_ret
-		kill -9 "$progress_bg_proc"
-		wait "$progress_bg_proc" &> /dev/null
-	} | progress '...'
-	[[ -f /tmp/preflight_ret ]] && preflight_ret=$(cat /tmp/preflight_ret)
-	rm /tmp/preflight_ret
-	if [[ -n $preflight_ret && $preflight_ret -gt 0 ]] && ! ( whiptail_preflight_err ); then
-		whiptail_cancel
-	fi
-fi
-
-percentage=0
-{
-	installer_progress_loop 'Checking that all required packages are installed and enabled...' & # Run progress bar to 98 in ~8 minutes while waiting for package installs
-	progress_bg_proc=$!
-	installer_prereq_packages
-	install_success=$?
-	kill -9 "$progress_bg_proc"
-	wait "$progress_bg_proc" &> /dev/null # Kill just sends signal, redirect output of wait to catch stdout
-	if [[ $install_success -gt 0 ]]; then
-		echo "Could not install packages required for setup, exiting now." >> "$setup_log" 2>&1
-		kill -SIGUSR1 "$setup_proc"; exit 1
-	fi
-} | progress '...'
-
-detect_cloud
-
-short_name=$(echo "$HOSTNAME" | awk -F. '{print $1}')
-
-if [[ $is_analyst ]]; then
-	MINION_ID=$(echo "${short_name}_workstation" | tr '[:upper:]' '[:lower:]')
-fi
-if [[ ! $is_analyst ]]; then
-	MINION_ID=$(echo "${short_name}_${install_type}" | tr '[:upper:]' '[:lower:]')
-fi
-export MINION_ID
-
-echo "MINION_ID = $MINION_ID" >> $setup_log 2>&1
-
-minion_type=$(get_minion_type)
-
-# Set any variables needed
-set_default_log_size >> $setup_log 2>&1
-
-if [[ $is_helix ]]; then
-	RULESETUP=${RULESETUP:-ETOPEN}
-	NSMSETUP=${NSMSETUP:-BASIC}
-	HNSENSOR=${HNSENSOR:-inherit}
-	MANAGERUPDATES=${MANAGERUPDATES:-0}
-fi
-
-if [[ $is_helix || ( $is_manager && $is_node ) ]]; then
-	RULESETUP=${RULESETUP:-ETOPEN}
-	NSMSETUP=${NSMSETUP:-BASIC}
-fi
-
-if [[ $is_manager && $is_node ]]; then
-	LSPIPELINEWORKERS=${LSPIPELINEWORKERS:-1}
-	LSPIPELINEBATCH=${LSPIPELINEBATCH:-125}
-	LSINPUTTHREADS=${LSINPUTTHREADS:-1}
-	LSPIPELINEWORKERS=${LSPIPELINEBATCH:-125}
-	NIDS=${NIDS:-Suricata}
-	ZEEKVERSION=${ZEEKVERSION:-ZEEK}
-fi
-
-if [[ $is_import ]]; then
-	PATCHSCHEDULENAME=${PATCHSCHEDULENAME:-auto}
-	MTU=${MTU:-1500}
-	RULESETUP=${RULESETUP:-ETOPEN}
-	NSMSETUP=${NSMSETUP:-BASIC}
-	HNSENSOR=${HNSENSOR:-inherit}
-	MANAGERUPDATES=${MANAGERUPDATES:-0}
-	MANAGERADV=${MANAGERADV:-BASIC}
-	INTERFACE=${INTERFACE:-bond0}
-	ZEEKVERSION=${ZEEKVERSION:-ZEEK}
-	NIDS=${NIDS:-Suricata}
-	RULESETUP=${RULESETUP:-ETOPEN}
-	GRAFANA=${GRAFANA:-0}
-	OSQUERY=${OSQUERY:-0}
-	WAZUH=${WAZUH:-0}
-	PLAYBOOK=${PLAYBOOK:-0}
-fi
-
-if [[ $is_airgap ]]; then
-	PATCHSCHEDULENAME=${PATCHSCHEDULENAME:-manual}
-	[[ ! $is_minion ]] && MANAGERUPDATES=${MANAGERUPDATES:-0} || MANAGERUPDATES=${MANAGERUPDATES:-1}
-fi
-
-# Start user prompts
-
-if [[ $is_helix ]]; then
-	collect_helix_key
-fi
-
-if [[ $is_helix || $is_sensor ]]; then
-	echo "Verifying all network devices are managed by Network Manager that should be" >> "$setup_log" 2>&1
-	check_network_manager_conf
-	set_network_dev_status_list
-	whiptail_sensor_nics
-fi
-
-if [[ $is_helix || $is_sensor || $is_import ]]; then
-	calculate_useable_cores
-fi
-
-if [[ ! $is_airgap && ! $is_import ]]; then
-	collect_patch_schedule
-fi
-
-if [[ $is_helix ||  $is_manager || $is_import ]]; then
-	collect_homenet_mngr
-fi
-
-#set base elasticsearch heap size
-if [[ $is_helix || $is_manager || $is_node || $is_import ]]; then
-	es_heapsize
-fi
-
-#set base logstash heap size
-if [[ $is_helix || $is_manager || $is_node || $is_import || $is_receiver ]]; then
-	ls_heapsize
-fi
-
-if [[ $is_manager && ! $is_eval ]]; then
-	whiptail_manager_adv
-	if [ "$MANAGERADV" = 'ADVANCED' ]; then
-	  if [ "$install_type" = 'MANAGER' ] || [ "$install_type" = 'MANAGERSEARCH' ]; then
-	  	collect_es_cluster_name
-	  fi
-	fi
-
-	whiptail_metadata_tool
-	
-	[[ $MANAGERADV == "ADVANCED" ]] && [[ $ZEEKVERSION == "ZEEK" ]] && whiptail_manager_adv_service_zeeklogs
-
-	# Don't run this function for now since Snort is not yet supported
-	# whiptail_nids
-	NIDS=Suricata
-	whiptail_rule_setup
-
-	if [ "$RULESETUP" != 'ETOPEN' ]; then
-		collect_oinkcode
-	fi
-fi
-
-if [[ $is_manager ]]; then
-	whiptail_enable_components
-
-	if [[ "$STRELKA" = 1 ]]; then
-		info "Enabling Strelka rules"
-		STRELKARULES=1
-	else
-		info "Disabling Strelka rules: STRELKA='$STRELKA'"
-	fi
-
-	collect_dockernet
-fi
-
-if [[ $is_manager || $is_import ]]; then
-	collect_webuser_inputs
-	get_redirect
-fi
-
-if [[ $is_distmanager ]]; then
-	collect_soremote_inputs
-fi
-
-if [[ $is_sensor && ! $is_eval ]]; then
-	[[ $is_manager ]] || collect_homenet_snsr
-	whiptail_sensor_config
-	if [ $NSMSETUP == 'ADVANCED' ]; then
-		if [[ $is_manager ]]; then
-			[[ $ZEEKVERSION == "ZEEK" ]] && whiptail_zeek_pins
-		else
-			whiptail_zeek_pins
-		fi
-		
-		whiptail_suricata_pins
-		collect_mtu
-	else
-		if [[ $is_node && $is_sensor && ! $is_eval ]]; then
-			PROCS=$(( lb_procs / 2 ))
-			if [ "$PROCS" -lt 1 ]; then PROCS=1; else PROCS=$PROCS; fi
-		else
-			PROCS=$lb_procs
-		fi
-
-		if [[ $is_manager ]]; then
-			[[ $ZEEKVERSION == "ZEEK" ]] && collect_zeek
-		else
-			collect_zeek
-		fi
-
-		collect_suri
-	fi
-fi
-
-[[ ( $is_iso || $is_analyst ) ]] && collect_ntp_servers
-
-if [[ ($is_node || $is_receiver) && ! $is_eval ]]; then
-	whiptail_node_advanced
-	if [ "$NODESETUP" == 'NODEADVANCED' ]; then
-		if [[ ! $is_receiver ]]; then 
-			collect_node_es_heap
-			collect_es_space_limit
-		fi
-		collect_node_ls_heap
-		collect_node_ls_pipeline_worker_count
-		collect_node_ls_pipeline_batch_size
-		collect_node_ls_input
-	else
-		if [[ ! $is_receiver ]]; then
-			NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE
-		fi
-		NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE
-		LSPIPELINEWORKERS=$num_cpu_cores
-		LSPIPELINEBATCH=125
-		LSINPUTTHREADS=1
-	fi
-fi
-
-if [ "$install_type" == 'FLEET' ]; then
-	collect_fleetuser_inputs
-	collect_fleet_custom_hostname_inputs
-else
-	FLEETNODEUSER=$WEBUSER
-	FLEETNODEPASSWD1=$WEBPASSWD1
-fi
-
-if [[ $is_manager || $is_import ]]; then collect_so_allow; fi
-
-# This block sets REDIRECTIT which is used by a function outside the below subshell
-set_redirect >> $setup_log 2>&1
-
-if [[ $is_minion ]] && ! check_manager_state; then
-	echo "Manager was not in a good state" >> "$setup_log" 2>&1
-	whiptail_manager_error
-fi
-
-whiptail_end_settings
-
-# From here on changes will be made.
-echo "1" > /root/accept_changes
-
-
-# Begin install
-{
-	# Set initial percentage to 0
-	export percentage=0
-
-	# Show initial progress message
-	set_progress_str 0 'Running initial configuration steps'
-
-	[[ ${#ntp_servers[@]} -gt 0 ]] && configure_ntp >> $setup_log 2>&1
-
-	if [[ ! $is_analyst ]]; then
-	    reserve_ports
-	fi
-	
-	set_path
-
-	if [[ $is_reinstall ]]; then
-		reinstall_init
-	fi
-
-	disable_auto_start
-
-	{
-		mark_version;
-		clear_manager;
-	} >> $setup_log 2>&1
-
-
-	if [[ $is_manager || $is_import ]]; then
-		{
-			generate_passwords;
-			secrets_pillar;
-		} >> $setup_log 2>&1	
-	fi
-
-	if [[ $is_manager || $is_import || $is_helix ]]; then
-		add_socore_user_manager >> $setup_log 2>&1
-	fi
-
-	if [[ $is_manager && ! $is_eval ]]; then
-		add_soremote_user_manager >> $setup_log 2>&1
-	fi
-    if [[ ! $is_analyst ]]; then
-        host_pillar >> $setup_log 2>&1
-    fi
-    if [[ $is_analyst ]]; then
-        analyst_workstation_pillar
-    fi
-	ntp_pillar >> $setup_log 2>&1
-
-
-	if [[ $is_minion || $is_import ]]; then
-		set_updates >> $setup_log 2>&1
-	fi
-
-	if [[ ( $is_manager || $is_import ) && $is_airgap ]]; then
-		info "Creating airgap repo"
-		create_repo >> $setup_log 2>&1
-		airgap_rules >> $setup_log 2>&1
-	fi
-
-	if [[ $is_minion ]]; then
-		set_progress_str 1 'Configuring firewall'
-		set_initial_firewall_policy >> $setup_log 2>&1
-	fi
-
-	set_progress_str 2 'Updating packages'
-	# Import the gpg keys
-	gpg_rpm_import >> $setup_log 2>&1
-	info "Disabling fastestmirror"
-	[[ $is_centos ]] && disable_fastestmirror
-	if [[ ! $is_airgap ]]; then
-		securityonion_repo >> $setup_log 2>&1
-		update_packages >> $setup_log 2>&1
-	else
-		airgap_repo >> $setup_log 2>&1
-	fi
-
-	if [[ $is_sensor || $is_helix || $is_import ]]; then
-		set_progress_str 3 'Generating sensor pillar'
-		generate_sensor_vars
-		sensor_pillar >> $setup_log 2>&1
-		if [[ $is_sensor || $is_helix ]]; then
-			steno_pillar >> $setup_log
-		fi
-	fi
-
-	if [[ $is_sensor || $is_helix ]]; then
-		set_progress_str 4 'Configuring sensor interface'
-		configure_network_sensor >> $setup_log 2>&1
-	fi
-
-	set_progress_str 5 'Installing Salt and dependencies'
-	saltify 2>> $setup_log
-    
-	if [[ ! $is_analyst ]]; then
-		set_progress_str 6 'Installing Docker and dependencies'
-		docker_install >> $setup_log 2>&1
-	fi
-
-	set_progress_str 7 'Generating patch pillar'
-	patch_pillar >> $setup_log 2>&1
-
-	set_progress_str 8 'Initializing Salt minion'
-	configure_minion "$minion_type" >> $setup_log 2>&1
-
-	if [[ ! $is_analyst ]]; then
-		check_sos_appliance >> $setup_log 2>&1
-	fi
-	
-	update_sudoers_for_testing >> $setup_log 2>&1
-
-	if [[ $is_manager || $is_helix || $is_import ]]; then
-		set_progress_str 9 'Configuring Salt master'
-		{
-			create_local_directories;
-			addtotab_generate_templates;
-			copy_salt_master_config;
-			setup_salt_master_dirs;
-			firewall_generate_templates;
-		} >> $setup_log 2>&1
-		
-		set_progress_str 10 'Updating sudoers file for soremote user'
-		update_sudoers >> $setup_log 2>&1
-		
-		set_progress_str 11 'Generating manager global pillar'
-		#minio_generate_keys
-		manager_global >> $setup_log 2>&1
-		
-		set_progress_str 12 'Generating manager pillar'
-		manager_pillar >> $setup_log 2>&1
-		zeek_logs_enabled >> $setup_log 2>&1
-	fi
-	
-	set_progress_str 16 'Running first Salt checkin'
-	salt_firstcheckin >> $setup_log 2>&1
-
-	if [[ $is_helix ]]; then
-		set_progress_str 17 'Generating the FireEye pillar'
-		fireeye_pillar >> $setup_log 2>&1
-	fi
-	
-	if [[ $is_node ]]; then
-		set_progress_str 18 'Setting node type'
-		set_node_type >> $setup_log 2>&1
-
-		if ! [[ $is_manager || $is_helix ]]; then
-			set_progress_str 19 'Generating search node pillar'
-			elasticsearch_pillar >> $setup_log 2>&1
-		fi
-	fi
-
-	if [[ ($is_node || $is_receiver) && !($is_manager || $is_helix) ]]; then
-		set_progress_str 19 'Generating logstash pillar'
-		logstash_pillar >> $setup_log 2>&1
-	fi
-
-	if [[ $is_idh ]]; then
-		# Write out services to minion pillar file
-		set_progress_str 19 'Generating IDH services pillar'
-		write_out_idh_services
-	fi
-
-
-	if [[ $is_minion ]]; then
-		set_progress_str 20 'Accepting Salt key on manager'
-		retry 20 10	accept_salt_key_remote "going to be accepted" >> $setup_log 2>&1
-	fi
-
-	if [[ $is_manager || $is_import || $is_helix ]]; then
-		set_progress_str 20 'Accepting Salt key'
-		retry 20 10 "salt-key -ya $MINION_ID" "going to be accepted" >> $setup_log 2>&1
-	fi
-
-	set_progress_str 21 'Copying minion pillars to manager'
-	copy_minion_tmp_files >> $setup_log 2>&1
-
-	if [[ $is_minion ]]; then
-		set_progress_str 22 'Checking if the Salt Minion needs to be updated'
-		salt-call state.apply -l info salt.minion >> $setup_log 2>&1
-	fi
-
-	if [[ $is_manager || $is_helix || $is_import ]]; then
-		set_progress_str 23 'Generating CA'
-		generate_ca >> $setup_log 2>&1
-	fi
-
-	if [[ ! $is_analyst ]]; then
-		set_progress_str 24 'Generating SSL'
-		generate_ssl >> $setup_log 2>&1
-	fi
-
-	if [[ $is_manager || $is_helix || $is_import ]]; then
-		set_progress_str 25 'Configuring firewall'
-		set_initial_firewall_policy >> $setup_log 2>&1
-		
-		# create these so the registry state can add so-registry to /opt/so/conf/so-status/so-status.conf
-		mkdir -p /opt/so/conf/so-status/ >> $setup_log 2>&1
-		touch /opt/so/conf/so-status/so-status.conf >> $setup_log 2>&1
-
-		if [[ "$setup_type" == 'iso' ]]; then
-			set_progress_str 26 'Copying containers from iso'
-		else
-			set_progress_str 26 'Downloading containers from the internet'
-		fi
-		import_registry_docker >> $setup_log 2>&1
-		salt-call state.apply -l info registry >> $setup_log 2>&1
-		docker_seed_registry # ~ 60% when finished
-		
-		set_progress_str 60 "$(print_salt_state_apply 'manager')"
-		salt-call state.apply -l info manager >> $setup_log 2>&1
-
-		echo "Executing so-elastic-auth..." >> $setup_log 2>&1
-		ELASTIC_AUTH_SKIP_HIGHSTATE=true bash /opt/so/saltstack/default/salt/common/tools/sbin/so-elastic-auth true >> $setup_log 2>&1
-		echo "Finished so-elastic-auth..." >> $setup_log 2>&1
-	fi
-
-	if [[ ! $is_analyst ]]; then
-		set_progress_str 61 "$(print_salt_state_apply 'firewall')"
-		salt-call state.apply -l info firewall >> $setup_log 2>&1
-	fi
-
-	if [[ $is_centos ]]; then
-		set_progress_str 61 'Installing Yum utilities'
-		salt-call state.apply -l info yum.packages >> $setup_log 2>&1
-	fi
-
-	if [[ ! $is_analyst ]]; then
-		set_progress_str 62 "$(print_salt_state_apply 'common')"
-		salt-call state.apply -l info common >> $setup_log 2>&1
-	fi
-
-	if [[ ! $is_helix && ! $is_receiver && ! $is_idh && ! $is_analyst ]]; then
-		set_progress_str 62 "$(print_salt_state_apply 'nginx')"
-		salt-call state.apply -l info nginx >> $setup_log 2>&1
-	fi
-
-	if [[ $is_manager || $is_helix || $is_import ]]; then
-		set_progress_str 63 "$(print_salt_state_apply 'idstools')"
-		create_local_nids_rules >> $setup_log 2>&1
-		salt-call state.apply -l info idstools >> $setup_log 2>&1
-
-		set_progress_str 63 "$(print_salt_state_apply 'suricata.manager')"
-		salt-call state.apply -l info suricata.manager >> $setup_log 2>&1
-	fi
-
-	if [[ $is_manager || $is_node || $is_import || $is_helix ]]; then
-		set_progress_str 64 "$(print_salt_state_apply 'elasticsearch')"
-		salt-call state.apply -l info elasticsearch >> $setup_log 2>&1
-	fi
-
-	if [[ $is_sensor || $is_import ]]; then
-		set_progress_str 65 "$(print_salt_state_apply 'pcap')"
-		salt-call state.apply -l info pcap >> $setup_log 2>&1
-	fi
-
-	if [[ $is_sensor || $is_import || $is_helix ]]; then
-		set_progress_str 66 "$(print_salt_state_apply 'suricata')"
-		salt-call state.apply -l info suricata >> $setup_log 2>&1
-
-		if [[ $(lookup_pillar "mdengine") == 'ZEEK' ]]; then
-			set_progress_str 67 "$(print_salt_state_apply 'zeek')"
-			salt-call state.apply -l info zeek >> $setup_log 2>&1
-		fi
-	fi
-
-	if [[ $is_node ]]; then
-		set_progress_str 68 "$(print_salt_state_apply 'curator')"
-		salt-call state.apply -l info curator >> $setup_log 2>&1
-	fi
-
-	if [[ $is_manager || $is_import ]]; then
-		set_progress_str 69 "$(print_salt_state_apply 'soc')"
-		salt-call state.apply -l info soc >> $setup_log 2>&1
-
-		set_progress_str 70 "$(print_salt_state_apply 'kibana')"
-		salt-call state.apply -l info kibana.so_config_load >> $setup_log 2>&1
-		salt-call state.apply -l info kibana.so_securitySolution_load >> $setup_log 2>&1
-		salt-call state.apply -l info kibana.so_dashboard_load >> $setup_log 2>&1
-
-		set_progress_str 70 "Setting up default Space in Kibana"
-		so-kibana-space-defaults >> $setup_log 2>&1
-	fi
-
-	if [[ "$PLAYBOOK" = 1 ]]; then
-		set_progress_str 71 "$(print_salt_state_apply 'playbook.db_init')"
-		salt-call state.apply -l info playbook.db_init >> $setup_log 2>&1
-
-		set_progress_str 71 "$(print_salt_state_apply 'playbook')"
-		salt-call state.apply -l info playbook >> $setup_log 2>&1
-
-		set_progress_str 71 "$(print_salt_state_apply 'playbook.automation_user_create')"
-		salt-call state.apply -l info playbook.automation_user_create >> $setup_log 2>&1
-	fi
-
-	if [[ $is_manager ]]; then
-		set_progress_str 72 "$(print_salt_state_apply 'elastalert')"
-		salt-call state.apply -l info elastalert >> $setup_log 2>&1
-
-		set_progress_str 73 "$(print_salt_state_apply 'soctopus')"
-		salt-call state.apply -l info soctopus >> $setup_log 2>&1
-
-		if [[ "$PLAYBOOK" = 1 ]]; then
-			set_progress_str 73 "Update playbook rules"
-			so-playbook-ruleupdate >> /root/setup_playbook_rule_update.log 2>&1 &
-		fi
-
-		if [[ "$GRAFANA" = 1 ]]; then
-			set_progress_str 74 "Installing InfluxDB and Grafana"
-			salt-call state.apply -l info influxdb >> $setup_log 2>&1
-			salt-call state.apply -l info grafana >> $setup_log 2>&1
-		fi
-
-	fi
-	
-	if [[ "$OSQUERY" = 1 ]]; then
-
-		set_progress_str 75 "$(print_salt_state_apply 'fleet.event_enable-fleet')"
-		salt-call state.apply -l info fleet.event_enable-fleet >> $setup_log 2>&1
-
-		set_progress_str 75 "$(print_salt_state_apply 'fleet')"
-		salt-call state.apply -l info fleet >> $setup_log 2>&1
-
-		set_progress_str 76 "$(print_salt_state_apply 'redis')"
-		salt-call state.apply -l info redis >> $setup_log 2>&1
-
-		if [[ $is_fleet_standalone && $FLEETCUSTOMHOSTNAME != '' ]]; then
-			set_progress_str 77 "$(print_salt_state_apply 'fleet.event_update-custom-hostname')"
-			pillar_override="{\"global\":{\"fleet_custom_hostname\": \"$FLEETCUSTOMHOSTNAME\"}}"
-			salt-call state.apply -l info fleet.event_update-custom-hostname pillar="$pillar_override" >> $setup_log 2>&1
-			rm -f /etc/pki/managerssl.crt
-			salt-call state.apply -l info ssl >> $setup_log 2>&1
-		fi
-	
-		set_progress_str 78 "$(print_salt_state_apply 'so-fleet-setup')"	
-		so-fleet-setup "$FLEETNODEUSER" "$FLEETNODEPASSWD1" >> $setup_log 2>&1
-
-	fi
-
-	if [[ $is_idh ]]; then
-		set_progress_str 79 "$(print_salt_state_apply 'idh')"
-		salt-call state.apply -l info idh >> $setup_log 2>&1
-
-	fi
-
-	if [[ "$WAZUH" = 1 ]]; then
-		set_progress_str 79 "$(print_salt_state_apply 'wazuh')"
-		salt-call state.apply -l info wazuh >> $setup_log 2>&1
-	fi
-
-	if [[ "$STRELKA" = 1 ]]; then
-		if [[ $is_sensor ]]; then
-			set_progress_str 81 "$(print_salt_state_apply 'strelka')"
-			salt-call state.apply -l info strelka >> $setup_log 2>&1
-		fi
-		if [[ "$STRELKARULES" = 1 ]]; then
-			logCmd /usr/sbin/so-yara-update
-		else
-			info "Skipping running yara update: STRELKARULES='$STRELKARULES'"
-		fi
-	fi
-
-	if [[ $is_manager || $is_import ]]; then		
-		set_progress_str 82 "$(print_salt_state_apply 'utility')"
-		salt-call state.apply -l info utility >> $setup_log 2>&1
-	fi
-
-	if [[ ( $is_helix || $is_manager || $is_node ) && ! $is_eval ]]; then
-		set_progress_str 83 "$(print_salt_state_apply 'logstash')"
-		salt-call state.apply -l info logstash >> $setup_log 2>&1
-
-		set_progress_str 84 "$(print_salt_state_apply 'filebeat')"
-		salt-call state.apply -l info filebeat >> $setup_log 2>&1
-	fi
-
-	if [[ ! $is_analyst ]]; then
-		set_progress_str 85 'Applying finishing touches'
-		filter_unused_nics >> $setup_log 2>&1
-		network_setup >> $setup_log 2>&1
-		so-ssh-harden >> $setup_log 2>&1
-	fi
-	
-	if [[ $is_manager || $is_import ]]; then
-		set_progress_str 87 'Adding user to SOC'
-		add_web_user >> $setup_log 2>&1
-	fi
-	
-	if [[ $is_analyst ]]; then
-		# Remove access to the manager from the analyst workstation
-		rm -rf /root/.ssh/so.key*
-	fi
-
-	set_progress_str 90 'Enabling checkin at boot'
-	checkin_at_boot >> $setup_log 2>&1
-
-	set_progress_str 95 'Verifying setup'
-	salt-call -l info state.highstate queue=True >> $setup_log 2>&1
-
-} | progress
-
-success=$(tail -10 $setup_log | grep Failed | awk '{ print $2}')
-if [[ $success != 0 ]]; then SO_ERROR=1; fi
-
-# Check entire setup log for errors or unexpected salt states and ensure cron jobs are not reporting errors to root's mailbox
-# Ignore "Status .* was not found" due to output from salt http.query or http.wait_for_successful_query states used with retry
-# Uncaught exception, closing connection|Exception in callback None - this is seen during influxdb / http.wait_for_successful_query state for ubuntu reinstall
-if grep -E "ERROR|Result: False" $setup_log | grep -qvE "Status .* was not found|An exception occurred in this state|Uncaught exception, closing connection|Exception in callback None|deprecation: ERROR" || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then
-	SO_ERROR=1
-	grep --color=never "ERROR" "$setup_log" | grep -qvE "Status .* was not found|An exception occurred in this state|Uncaught exception, closing connection|Exception in callback None" > "$error_log"
-fi
-
-if [[ -n $SO_ERROR ]]; then 
-	echo "Errors detected during setup; skipping post-setup steps to allow for analysis of failures." >> $setup_log 2>&1
-
-	SKIP_REBOOT=1
-	whiptail_setup_failed
-else 
-	echo "Successfully completed setup! Continuing with post-installation steps" >> $setup_log 2>&1
-	{
-		export percentage=95 # set to last percentage used in previous subshell
-		if [[ -n $ALLOW_ROLE && -n $ALLOW_CIDR ]]; then 
-			set_progress_str 96 "Stopping SOC prior to adjusting firewall rules"
-			so-soc-stop # Stop SOC so it doesn't accept external requests prior to the reboot
-			
-			set_progress_str 97 "Running so-allow -${ALLOW_ROLE} for ${ALLOW_CIDR}"
-			IP=$ALLOW_CIDR so-allow -$ALLOW_ROLE >> $setup_log 2>&1
-		fi
-
-		if [[ $is_manager ]]; then
-			set_progress_str 98 "Generating archive for setup directory"
-			generate_repo_tarball >> "$setup_log" 2>&1
-		fi
-
-		if [[ -n $LEARN_LOGSCAN_ENABLE ]]; then
-			set_progress_str 99 'Enabling logscan'
-			so-learn enable logscan --apply >> $setup_log 2>&1
-		fi
-
-		if [[ -n $ENDGAMEHOST ]]; then
-			set_progress_str 99 'Configuring firewall for Endgame SMP'
-			so-firewall --apply includehost endgame $ENDGAMEHOST   >> $setup_log 2>&1
-		fi
-
-	} | whiptail_gauge_post_setup "Running post-installation steps..."
-
-	echo "Post-installation steps have completed. Awaiting user input to clean up installer." >> $setup_log 2>&1
-	whiptail_setup_complete
-	[[ $setup_type != 'iso' && ! $is_idh ]] && whiptail_ssh_warning
-fi
-
-install_cleanup >> "$setup_log" 2>&1
-
-if [[ -z $SKIP_REBOOT ]]; then shutdown -r now; else exit; fi

From 8a5a58c64788d544e2dd9cd866b8b1a333ce3d79 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 14 Sep 2022 12:21:21 -0400
Subject: [PATCH 05/16] Fix so-minion error for setup

---
 salt/common/tools/sbin/so-minion | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/salt/common/tools/sbin/so-minion b/salt/common/tools/sbin/so-minion
index 58f04ab77..cb1c4db3e 100755
--- a/salt/common/tools/sbin/so-minion
+++ b/salt/common/tools/sbin/so-minion
@@ -5,8 +5,9 @@
 # https://securityonion.net/license; you may not use this file except in compliance with the
 # Elastic License 2.0.
 
-
+if [ $OPERATION != 'setup' ]; then
 . /usr/sbin/so-common
+fi
 
 if [[ $# -lt 1 ]]; then
   echo "Usage: $0 -o=<operation> -m=[id]"

From 8049f9b9e4989c2c17f5291e693dcf5b94608ed9 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 14 Sep 2022 12:22:10 -0400
Subject: [PATCH 06/16] Fix so-minion error for setup

---
 salt/common/tools/sbin/so-minion | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/salt/common/tools/sbin/so-minion b/salt/common/tools/sbin/so-minion
index cb1c4db3e..89c00bb3c 100755
--- a/salt/common/tools/sbin/so-minion
+++ b/salt/common/tools/sbin/so-minion
@@ -164,11 +164,11 @@ function add_sensor_to_minion() {
 	echo "  mtu: 9000" >> $PILLARFILE
 	echo "zeek:" >> $PILLARFILE
 	echo "    config:" >> $PILLARFILE
-	echo "        lb_procs: $CORECOUNT" >> $PILLARFILE
+	echo "        lb_procs: '$CORECOUNT'" >> $PILLARFILE
 	echo "suricata:" >> $PILLARFILE
 	echo "    config:" >> $PILLARFILE
 	echo "        af-packet:" >> $PILLARFILE
-	echo "            threads: $CORECOUNT" >> $PILLARFILE
+	echo "            threads: '$CORECOUNT'" >> $PILLARFILE
 }
 
 function createSTANDALONE() {

From c60afba45089571d2b0e5fa24f45a1cf9e670fd9 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 14 Sep 2022 12:30:22 -0400
Subject: [PATCH 07/16] Fix core count

---
 setup/so-setup | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/setup/so-setup b/setup/so-setup
index d9680ac6c..8dba4676a 100755
--- a/setup/so-setup
+++ b/setup/so-setup
@@ -510,7 +510,8 @@ if ! [[ -f $install_opt_file ]]; then
 		export MAINIP=$MAINIP
 		export PATCHSCHEDULENAME=$PATCHSCHEDULENAME
 		export INTERFACE="bond0"
-		so-minion -o=setup 
+		export CORECOUNT=$lb_procs
+		logCmd "so-minion -o=setup" 
 		title "Creating Global SLS"
 
 		if [[ $is_airgap ]]; then

From 74a8bd17ea9fff6215fc494bcd44b8d2aaa6762d Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 14 Sep 2022 13:56:17 -0400
Subject: [PATCH 08/16] Fix Suricata

---
 salt/suricata/defaults.yaml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml
index 4e4d28a87..0ae541ffd 100644
--- a/salt/suricata/defaults.yaml
+++ b/salt/suricata/defaults.yaml
@@ -148,11 +148,6 @@ suricata:
           enabled: "no"
           facility: local5
           format: "[%i] <%d> -- "
-    pcap:
-      - interface: eth0
-      - interface: default
-    pcap-file:
-      checksum-checks: auto
     app-layer:
       protocols:
         krb5:

From ee59822097d1269ec516bf31e135612f2770ee4b Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 14 Sep 2022 14:15:50 -0400
Subject: [PATCH 09/16] Fix Suricata

---
 salt/suricata/suricata_config.map.jinja | 58 -------------------------
 1 file changed, 58 deletions(-)

diff --git a/salt/suricata/suricata_config.map.jinja b/salt/suricata/suricata_config.map.jinja
index da14f8d9a..2b14103a5 100644
--- a/salt/suricata/suricata_config.map.jinja
+++ b/salt/suricata/suricata_config.map.jinja
@@ -1,70 +1,12 @@
 {% import_yaml 'suricata/defaults.yaml' as suricata_defaults with context %}
 {% import_yaml 'suricata/suricata_meta.yaml' as suricata_meta with context %}
-{% from 'suricata/afpacket.map.jinja' import afpacket %}
 {% set suricata_pillar = salt['pillar.get']('suricata:config', {}) %}
 {% set default_evelog_index = [] %}
 {% set default_filestore_index = [] %}
 {% set surimeta_evelog_index = [] %}
 {% set surimeta_filestore_index = [] %}
 
-{% if salt['pillar.get']('sensor:hnsensor') %}
-  {% load_yaml as homenet %}
-HOME_NET: "[{{salt['pillar.get']('sensor:hnsensor')}}]"
-  {% endload %}
-{% else %}
-  {% load_yaml as homenet %}
-HOME_NET: "[{{salt['pillar.get']('global:hnmanager', '')}}]"
-  {% endload %}
-{% endif %}
-
 {% set hardware_header = 15 %}
-{% set default_packet_size = salt['grains.filter_by']({
-    '*_eval': {
-      'default-packet-size': salt['pillar.get']('sensor:mtu', 1500) + hardware_header,
-    },
-    '*_helixsensor': {
-      'default-packet-size': salt['pillar.get']('sensor:mtu', 9000) + hardware_header,
-    },
-    '*': {
-      'default-packet-size': salt['pillar.get']('sensor:mtu', 1500) + hardware_header,
-    },
-},grain='id') %}
 
-{# Find the index of eve-log so it can be updated later #}
-{% for li in suricata_defaults.suricata.config.outputs %}
-  {% if 'eve-log' in li.keys() %}
-    {% do default_evelog_index.append(loop.index0) %}
-  {% endif %}
-  {% if 'file-store' in li.keys() %}
-    {% do default_filestore_index.append(loop.index0) %}
-  {% endif %}
-{% endfor %}
-{% set default_evelog_index = default_evelog_index[0]  %}
-{% set default_filestore_index = default_filestore_index[0]  %}
-
-{# Find the index of eve-log so it can be grabbed later #}
-{% for li in suricata_meta.suricata.config.outputs %}
-  {% if 'eve-log' in li.keys() %}
-    {% do surimeta_evelog_index.append(loop.index0) %}
-  {% endif %}
-  {% if 'file-store' in li.keys() %}
-    {% do surimeta_filestore_index.append(loop.index0) %}
-  {% endif %}
-{% endfor %}
-{% set surimeta_evelog_index = surimeta_evelog_index[0] %}
-{% set surimeta_filestore_index = surimeta_filestore_index[0] %}
-
-{% if salt['pillar.get']('global:mdengine', 'ZEEK') == 'SURICATA' %}
-  {% do suricata_defaults.suricata.config.outputs[default_evelog_index]['eve-log'].types.extend(suricata_meta.suricata.config.outputs[surimeta_evelog_index]['eve-log'].types) %}
-  {% do suricata_defaults.suricata.config.outputs[default_filestore_index]['file-store'].update({'enabled':suricata_meta.suricata.config.outputs[surimeta_filestore_index]['file-store']['enabled']}) %}
-{% endif %}
-
-{% do suricata_defaults.suricata.config.update(default_packet_size) %}
-{% do suricata_defaults.suricata.config.update(afpacket) %}
-{% do suricata_defaults.suricata.config.vars['address-groups'].update(homenet) %}
-{% if salt['pillar.get']('sensor:suriprocs', salt['pillar.get']('sensor:suripins', {})) %}
-  {% from 'suricata/threading.map.jinja' import cpu_affinity with context %}
-  {% do suricata_defaults.suricata.config.threading.update(cpu_affinity) %}
-{% endif %}
 
 {% do salt['defaults.merge'](suricata_defaults.suricata.config, suricata_pillar, in_place=True) %}
\ No newline at end of file

From 90ed4fd4cbe538278f166041c748010fb03510b6 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 14 Sep 2022 14:18:10 -0400
Subject: [PATCH 10/16] Fix Suricata

---
 salt/suricata/suricata_config.map.jinja | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/salt/suricata/suricata_config.map.jinja b/salt/suricata/suricata_config.map.jinja
index 2b14103a5..d6c62f258 100644
--- a/salt/suricata/suricata_config.map.jinja
+++ b/salt/suricata/suricata_config.map.jinja
@@ -1,12 +1,7 @@
 {% import_yaml 'suricata/defaults.yaml' as suricata_defaults with context %}
 {% import_yaml 'suricata/suricata_meta.yaml' as suricata_meta with context %}
 {% set suricata_pillar = salt['pillar.get']('suricata:config', {}) %}
-{% set default_evelog_index = [] %}
-{% set default_filestore_index = [] %}
-{% set surimeta_evelog_index = [] %}
-{% set surimeta_filestore_index = [] %}
 
-{% set hardware_header = 15 %}
 
 
 {% do salt['defaults.merge'](suricata_defaults.suricata.config, suricata_pillar, in_place=True) %}
\ No newline at end of file

From 30a469ea63b6445afb68f00611b50b73258dbf69 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 14 Sep 2022 14:36:13 -0400
Subject: [PATCH 11/16] Update afpacket.map.jinja

---
 salt/suricata/afpacket.map.jinja | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/salt/suricata/afpacket.map.jinja b/salt/suricata/afpacket.map.jinja
index a6c390abb..2c575c456 100644
--- a/salt/suricata/afpacket.map.jinja
+++ b/salt/suricata/afpacket.map.jinja
@@ -1,15 +1,14 @@
+{% import_yaml 'suricata/defaults.yaml' as suricata_defaults with context %}
+{% set suricata_pillar = pillar.suricata %}
+{% set surimerge = salt['defaults.merge'](suricata_defaults, suricata_pillar, in_place=False)
 {% load_yaml as afpacket %}
 af-packet:
-  - interface: {{ salt['pillar.get']('sensor:interface', 'bond0') }}
-    cluster-id: 59
-    cluster-type: cluster_flow
-    defrag: yes
-    use-mmap: yes
-    threads: {{ salt['pillar.get']('sensor:suriprocs', salt['pillar.get']('sensor:suripins') | length) }}
-    tpacket-v3: yes
-    ring-size: {{ salt['pillar.get']('sensor:suriringsize', '5000') }}
-  - interface: default
-    #threads: auto
-    #use-mmap: no
-    #tpacket-v3: yes
+  - interface: {{ surimerge.suricata.config.af-packet.interface }}
+    cluster-id: {{ surimerge.suricata.config.af-packet.cluster-id }}
+    cluster-type: {{ surimerge.suricata.config.af-packet.cluster-type }}
+    defrag: {{ surimerge.suricata.config.af-packet.defrag }}
+    use-mmap: {{ surimerge.suricata.config.af-packet.use-mmap }}
+    threads: {{ surimerge.suricata.config.af-packet.threads }}
+    tpacket-v3: {{ surimerge.suricata.config.af-packet.tpacket-v3 }}
+    ring-size: {{ surimerge.suricata.config.af-packet.ring-size }}
 {% endload %}

From f8c1571a91a28468218500b38d56c436951f17d4 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 14 Sep 2022 14:43:04 -0400
Subject: [PATCH 12/16] Update suricata_config.map.jinja

---
 salt/suricata/suricata_config.map.jinja | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/salt/suricata/suricata_config.map.jinja b/salt/suricata/suricata_config.map.jinja
index d6c62f258..5790832ef 100644
--- a/salt/suricata/suricata_config.map.jinja
+++ b/salt/suricata/suricata_config.map.jinja
@@ -1,7 +1,17 @@
 {% import_yaml 'suricata/defaults.yaml' as suricata_defaults with context %}
-{% import_yaml 'suricata/suricata_meta.yaml' as suricata_meta with context %}
-{% set suricata_pillar = salt['pillar.get']('suricata:config', {}) %}
+{% set suricata_pillar = pillar.suricata %}
+{% set surimerge = salt['defaults.merge'](suricata_defaults, suricata_pillar, in_place=False)
+{% load_yaml as afpacket %}
+af-packet:
+  - interface: {{ surimerge.suricata.config.af-packet.interface }}
+    cluster-id: {{ surimerge.suricata.config.af-packet.cluster-id }}
+    cluster-type: {{ surimerge.suricata.config.af-packet.cluster-type }}
+    defrag: {{ surimerge.suricata.config.af-packet.defrag }}
+    use-mmap: {{ surimerge.suricata.config.af-packet.use-mmap }}
+    threads: {{ surimerge.suricata.config.af-packet.threads }}
+    tpacket-v3: {{ surimerge.suricata.config.af-packet.tpacket-v3 }}
+    ring-size: {{ surimerge.suricata.config.af-packet.ring-size }}
+{% endload %}
+{% do suricata_defaults.suricata.config.update({'af-packet': afpacket}) %}
 
-
-
-{% do salt['defaults.merge'](suricata_defaults.suricata.config, suricata_pillar, in_place=True) %}
\ No newline at end of file
+{% do salt['defaults.merge'](suricata_defaults.suricata.config, suricata_pillar, in_place=True) %}

From 8528645c2ccf21a8f0891b12aafeb7a625757d25 Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Wed, 14 Sep 2022 15:02:55 -0400
Subject: [PATCH 13/16] Update suricata_config.map.jinja

---
 salt/suricata/suricata_config.map.jinja | 29 ++++++++++++++++---------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/salt/suricata/suricata_config.map.jinja b/salt/suricata/suricata_config.map.jinja
index 5790832ef..cf576179e 100644
--- a/salt/suricata/suricata_config.map.jinja
+++ b/salt/suricata/suricata_config.map.jinja
@@ -1,17 +1,26 @@
 {% import_yaml 'suricata/defaults.yaml' as suricata_defaults with context %}
 {% set suricata_pillar = pillar.suricata %}
-{% set surimerge = salt['defaults.merge'](suricata_defaults, suricata_pillar, in_place=False)
+{% set surimerge = salt['defaults.merge'](suricata_defaults, suricata_pillar, in_place=False) %}
 {% load_yaml as afpacket %}
 af-packet:
-  - interface: {{ surimerge.suricata.config.af-packet.interface }}
-    cluster-id: {{ surimerge.suricata.config.af-packet.cluster-id }}
-    cluster-type: {{ surimerge.suricata.config.af-packet.cluster-type }}
-    defrag: {{ surimerge.suricata.config.af-packet.defrag }}
-    use-mmap: {{ surimerge.suricata.config.af-packet.use-mmap }}
-    threads: {{ surimerge.suricata.config.af-packet.threads }}
-    tpacket-v3: {{ surimerge.suricata.config.af-packet.tpacket-v3 }}
-    ring-size: {{ surimerge.suricata.config.af-packet.ring-size }}
+  - interface: {{ surimerge.suricata.config['af-packet'].interface }}
+    cluster-id: {{ surimerge.suricata.config['af-packet']['cluster-id'] }}
+    cluster-type: {{ surimerge.suricata.config['af-packet']['cluster-type'] }}
+    defrag: {{ surimerge.suricata.config['af-packet'].defrag }}
+    use-mmap: {{ surimerge.suricata.config['af-packet']['use-mmap'] }}
+    threads: {{ surimerge.suricata.config['af-packet'].threads }}
+    tpacket-v3: {{ surimerge.suricata.config['af-packet']['tpacket-v3'] }}
+    ring-size: {{ surimerge.suricata.config['af-packet']['ring-size'] }}
 {% endload %}
 {% do suricata_defaults.suricata.config.update({'af-packet': afpacket}) %}
 
-{% do salt['defaults.merge'](suricata_defaults.suricata.config, suricata_pillar, in_place=True) %}
+{% load_yaml as outputs %}
+outputs:
+  - fast:
+    enabled: {{ surimerge.suricata.config.outputs.fast.enabled }}
+    filename: {{ surimerge.suricata.config.outputs.fast.filename }}
+    append: {{ surimerge.suricata.config.outputs.fast.append }}
+  - eve-log
+    enabled: {{ surimerge.suricata.config.outputs['eve-log'].enabled }}
+{% endload %}
+{% do suricata_defaults.suricata.config.update({'outputs': outputs}) %}

From 4cb955fe8d6ea266490ab20a72d5c68b7554fd24 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 15 Sep 2022 10:35:59 -0400
Subject: [PATCH 14/16] jinja for the suricata outputs

---
 salt/suricata/suricata_config.map.jinja | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/salt/suricata/suricata_config.map.jinja b/salt/suricata/suricata_config.map.jinja
index cf576179e..07382498f 100644
--- a/salt/suricata/suricata_config.map.jinja
+++ b/salt/suricata/suricata_config.map.jinja
@@ -1,6 +1,7 @@
 {% import_yaml 'suricata/defaults.yaml' as suricata_defaults with context %}
 {% set suricata_pillar = pillar.suricata %}
 {% set surimerge = salt['defaults.merge'](suricata_defaults, suricata_pillar, in_place=False) %}
+
 {% load_yaml as afpacket %}
 af-packet:
   - interface: {{ surimerge.suricata.config['af-packet'].interface }}
@@ -15,12 +16,8 @@ af-packet:
 {% do suricata_defaults.suricata.config.update({'af-packet': afpacket}) %}
 
 {% load_yaml as outputs %}
-outputs:
-  - fast:
-    enabled: {{ surimerge.suricata.config.outputs.fast.enabled }}
-    filename: {{ surimerge.suricata.config.outputs.fast.filename }}
-    append: {{ surimerge.suricata.config.outputs.fast.append }}
-  - eve-log
-    enabled: {{ surimerge.suricata.config.outputs['eve-log'].enabled }}
+{% for le, ld in surimerge.suricata.config.outputs.items() %}
+  - {{ le }}: {{ ld }}
+{% endfor %}
 {% endload %}
 {% do suricata_defaults.suricata.config.update({'outputs': outputs}) %}

From 845d2e33bd07dde6261c29a51f2beacd9ea77b96 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Thu, 15 Sep 2022 10:44:39 -0400
Subject: [PATCH 15/16] remove dupe afpacket

---
 salt/suricata/suricata_config.map.jinja | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/salt/suricata/suricata_config.map.jinja b/salt/suricata/suricata_config.map.jinja
index 07382498f..2a03f5fbb 100644
--- a/salt/suricata/suricata_config.map.jinja
+++ b/salt/suricata/suricata_config.map.jinja
@@ -3,15 +3,14 @@
 {% set surimerge = salt['defaults.merge'](suricata_defaults, suricata_pillar, in_place=False) %}
 
 {% load_yaml as afpacket %}
-af-packet:
-  - interface: {{ surimerge.suricata.config['af-packet'].interface }}
-    cluster-id: {{ surimerge.suricata.config['af-packet']['cluster-id'] }}
-    cluster-type: {{ surimerge.suricata.config['af-packet']['cluster-type'] }}
-    defrag: {{ surimerge.suricata.config['af-packet'].defrag }}
-    use-mmap: {{ surimerge.suricata.config['af-packet']['use-mmap'] }}
-    threads: {{ surimerge.suricata.config['af-packet'].threads }}
-    tpacket-v3: {{ surimerge.suricata.config['af-packet']['tpacket-v3'] }}
-    ring-size: {{ surimerge.suricata.config['af-packet']['ring-size'] }}
+- interface: {{ surimerge.suricata.config['af-packet'].interface }}
+  cluster-id: {{ surimerge.suricata.config['af-packet']['cluster-id'] }}
+  cluster-type: {{ surimerge.suricata.config['af-packet']['cluster-type'] }}
+  defrag: {{ surimerge.suricata.config['af-packet'].defrag }}
+  use-mmap: {{ surimerge.suricata.config['af-packet']['use-mmap'] }}
+  threads: {{ surimerge.suricata.config['af-packet'].threads }}
+  tpacket-v3: {{ surimerge.suricata.config['af-packet']['tpacket-v3'] }}
+  ring-size: {{ surimerge.suricata.config['af-packet']['ring-size'] }}
 {% endload %}
 {% do suricata_defaults.suricata.config.update({'af-packet': afpacket}) %}
 

From 0351ef4ff59c01eebf0b50575a0101b56a444d1c Mon Sep 17 00:00:00 2001
From: Mike Reeves <mike.reeves@securityonionsolutions.com>
Date: Thu, 15 Sep 2022 10:48:08 -0400
Subject: [PATCH 16/16] Fix Suricata analyzers list

---
 salt/suricata/defaults.yaml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/salt/suricata/defaults.yaml b/salt/suricata/defaults.yaml
index 0ae541ffd..cf7db31af 100644
--- a/salt/suricata/defaults.yaml
+++ b/salt/suricata/defaults.yaml
@@ -245,6 +245,14 @@ suricata:
           enabled: "yes"
         sip:
           enabled: "yes"
+        rfb:
+          enabled: 'yes'
+          detection-ports:
+            dp: 5900, 5901, 5902, 5903, 5904, 5905, 5906, 5907, 5908, 5909
+        mqtt:
+          enabled: 'no'
+        http2:
+          enabled: 'no'
     asn1-max-frames: 256
     run-as:
       user: suricata