Merge branch 'dev' into kilo

pillar/logstash/search.sls

@@ -7,6 +7,7 @@ logstash:
         - so/9000_output_zeek.conf.jinja
         - so/9002_output_import.conf.jinja
         - so/9034_output_syslog.conf.jinja
+        - so/9050_output_filebeatmodules.conf.jinja
         - so/9100_output_osquery.conf.jinja  
         - so/9400_output_suricata.conf.jinja
         - so/9500_output_beats.conf.jinja

pillar/zeek/init.sls

@@ -52,5 +52,4 @@ zeek:
       - frameworks/signatures/detect-windows-shells
     redef:
       - LogAscii::use_json = T;
-      - LogAscii::json_timestamps = JSON::TS_ISO8601;
       - CaptureLoss::watch_interval = 5 mins;

salt/common/tools/sbin/so-checkin

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-salt-call state.highstate
+salt-call state.highstate -linfo 

salt/common/tools/sbin/so-filebeat-module-setup

@@ -0,0 +1,67 @@
+{%- set mainint = salt['pillar.get']('host:mainint') %}
+{%- set MYIP = salt['grains.get']('ip_interfaces:' ~ mainint)[0] %}
+
+#!/bin/bash
+# Copyright 2014,2015,2016,2017,2018,2019 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+default_conf_dir=/opt/so/conf
+ELASTICSEARCH_HOST="{{ MYIP }}"
+ELASTICSEARCH_PORT=9200
+#ELASTICSEARCH_AUTH=""
+
+# Define a default directory to load pipelines from
+FB_MODULE_YML="/usr/share/filebeat/module-setup.yml"
+
+
+# Wait for ElasticSearch to initialize
+echo -n "Waiting for ElasticSearch..."
+COUNT=0
+ELASTICSEARCH_CONNECTED="no"
+while [[ "$COUNT" -le 240 ]]; do
+        curl -k --output /dev/null --silent --head --fail -L https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"
+        if [ $? -eq 0 ]; then
+                ELASTICSEARCH_CONNECTED="yes"
+                echo "connected!"
+                break
+        else
+                ((COUNT+=1))
+                sleep 1
+                echo -n "."
+        fi
+done
+if [ "$ELASTICSEARCH_CONNECTED" == "no" ]; then
+        echo
+        echo -e "Connection attempt timed out.  Unable to connect to ElasticSearch.  \nPlease try: \n  -checking log(s) in /var/log/elasticsearch/\n  -running 'sudo docker ps' \n  -running 'sudo so-elastic-restart'"
+        echo
+fi
+echo "Testing to see if the pipelines are already applied"
+ESVER=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT" |jq .version.number |tr -d \")
+PIPELINES=$(curl -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_PORT"/_ingest/pipeline/filebeat-$ESVER-suricata-eve-pipeline | jq . | wc -c)
+
+if [[ "$PIPELINES" -lt 5 ]]; then
+  echo "Setting up ingest pipeline(s)"
+
+  for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler
+  do
+    echo "Loading $MODULE"
+    docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML
+    sleep 2
+  done
+else
+  exit 0
+fi
+
+

salt/common/tools/sbin/so-zeek-logs

@@ -14,7 +14,6 @@ whiptail_manager_adv_service_zeeklogs() {
   "conn" "Connection Logging" ON \
   "dce_rpc" "RPC Logs" ON \
   "dhcp" "DHCP Logs" ON \
-  "dhcpv6" "DHCP IPv6 Logs" ON \
   "dnp3" "DNP3 Logs" ON \
   "dns" "DNS Logs" ON \
   "dpd" "DPD Logs" ON \
@@ -25,25 +24,20 @@ whiptail_manager_adv_service_zeeklogs() {
   "irc" "IRC Chat Logs" ON \
   "kerberos" "Kerberos Logs" ON \
   "modbus" "MODBUS Logs" ON \
-  "mqtt" "MQTT Logs" ON \
   "notice" "Zeek Notice Logs" ON \
   "ntlm" "NTLM Logs" ON \
-  "openvpn" "OPENVPN Logs" ON \
   "pe" "PE Logs" ON \
   "radius" "Radius Logs" ON \
   "rfb" "RFB Logs" ON \
   "rdp" "RDP Logs" ON \
-  "signatures" "Signatures Logs" ON \
   "sip" "SIP Logs" ON \
   "smb_files" "SMB Files Logs" ON \
   "smb_mapping" "SMB Mapping Logs" ON \
   "smtp" "SMTP Logs" ON \
   "snmp" "SNMP Logs" ON \
-  "software" "Software Logs" ON \
   "ssh" "SSH Logs" ON \
   "ssl" "SSL Logs" ON \
   "syslog" "Syslog Logs" ON \
-  "telnet" "Telnet Logs" ON \
   "tunnel" "Tunnel Logs" ON \
   "weird" "Zeek Weird Logs" ON \
   "mysql" "MySQL Logs" ON \

salt/common/tools/sbin/soup

@@ -319,14 +319,6 @@ masterunlock() {
   fi
 }
 
-preupgrade_changes_2.3.50_repo() {
-    # We made repo changes in 2.3.50 and this prepares for that on upgrade
-    echo "Checking to see if 2.3.50 repo changes are needed."
-
-    [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50_repo
-    true
-}
-
 preupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Checking to see if changes are needed."
@@ -517,8 +509,8 @@ up_2.3.2X_to_2.3.30() {
   INSTALLEDVERSION=2.3.30
 }
 
-up_2.3.3X_to_2.3.50_repo() {
-  echo "Performing 2.3.50 repo actions."
+upgrade_to_2.3.50_repo() {
+  echo "Performing repo changes."
   if [[ "$OS" == "centos" ]]; then
     # Import GPG Keys
     gpg_rpm_import
@@ -787,6 +779,12 @@ main() {
   verify_latest_update_script
   echo ""
 
+  echo "Stopping Salt Master service."
+  systemctl stop salt-master
+  echo ""
+
+  upgrade_to_2.3.50_repo
+
   echo "Generating new repo archive"
   generate_and_clean_tarballs
   if [ -f /usr/sbin/so-image-common ]; then

salt/elasticsearch/files/elasticsearch.yml

@@ -38,7 +38,7 @@ xpack.security.authc:
     authz_exception: true
 {% endif %}
 node.name: {{ grains.host }}
-script.max_compilations_rate: 1000/1m
+script.max_compilations_rate: 20000/1m
 {%- if TRUECLUSTER is sameas true %}
   {%- if grains.role == 'so-manager' %}
     {%- if salt['pillar.get']('nodestab', {}) %}

salt/elasticsearch/files/ingest/ossec

@@ -63,7 +63,8 @@
     { "rename":         { "field": "fields.module",              "target_field": "event.module",                  "ignore_failure": true, "ignore_missing": true  } },
     { "pipeline":      { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon"  }  },
     { "pipeline":      { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" }  },
-    { "set":           { "if": "ctx.rule != null && ctx.rule.name != null",   "field": "event.dataset", "value": "alert", "override": true }  },
+    { "set":           { "if": "ctx.rule != null && ctx.rule.name != null",   "field": "event.dataset", "value": "ossec.alert", "override": true }  },
+    { "set":           { "if": "ctx.rule != null && ctx.rule.name != null",   "field": "event.kind", "value": "alert", "override": true }  },
     { "pipeline":       { "name": "common" } }    
   ]
 }

salt/elasticsearch/files/ingest/strelka.file

@@ -53,7 +53,8 @@
     { "set":         { "if": "ctx.exiftool?.FileDirectory != null", "field": "file.directory",        "value": "{{exiftool.FileDirectory}}", "ignore_failure": true }},
     { "set":         { "if": "ctx.exiftool?.Subsystem != null", "field": "host.subsystem",        "value": "{{exiftool.Subsystem}}", "ignore_failure": true }},
     { "set":         { "if": "ctx.scan?.yara?.matches != null", "field": "rule.name",        "value": "{{scan.yara.matches.0}}" }},
-    { "set":         { "if": "ctx.scan?.yara?.matches != null", "field": "dataset",        "value": "alert", "override": true }},
+    { "set":         { "if": "ctx.scan?.yara?.matches != null", "field": "dataset",        "value": "strelka.alert", "override": true }},
+    { "set":         { "if": "ctx.scan?.yara?.matches != null", "field": "event.kind",        "value": "alert", "override": true }},
     { "rename":         { "field": "file.flavors.mime", "target_field": "file.mime_type",        "ignore_missing": true }},
     { "set":         { "if": "ctx.rule?.name != null && ctx.rule?.score == null",   "field": "event.severity", "value": 3, "override": true }  },
     { "convert" :    { "if": "ctx.rule?.score != null", "field" : "rule.score","type": "integer"}},

salt/elasticsearch/templates/so/so-common-template.json

@@ -1,5 +1,5 @@
 {
-  "index_patterns": ["so-ids-*", "so-firewall-*", "so-syslog-*", "so-zeek-*", "so-import-*", "so-ossec-*", "so-strelka-*", "so-beats-*", "so-osquery-*","so-playbook-*"],
+  "index_patterns": ["so-*"],
   "version":50001,
   "order":10,
   "settings":{
@@ -527,11 +527,19 @@
             "version":{
               "type":"long"
             }
-        }
-      },
+          }
+        },
         "x509":{
           "type":"object",
           "dynamic": true
+        },
+	"suricata":{
+          "type":"object",
+          "dynamic": true
+        },
+	"zeek":{
+          "type":"object",
+          "dynamic": true
         }
      }
   }

salt/filebeat/etc/filebeat.yml

@@ -73,7 +73,13 @@ logging.files:
 # Set to true to log messages in json format.
 #logging.json: false
 
+
+
 #==========================  Modules configuration ============================
+filebeat.config.modules:
+  enabled: true
+  path: ${path.config}/modules.d/*.yml
+
 filebeat.modules:
 #=========================== Filebeat prospectors =============================
 
@@ -185,7 +191,6 @@ filebeat.inputs:
   fields_under_root: true
   clean_removed: false
   close_removed: false
-
   {%- if STRELKAENABLED == 1 %}
 - type: log
   paths:

salt/filebeat/etc/module-setup.yml

@@ -0,0 +1,10 @@
+{%- if grains['role'] in ['so-managersearch', 'so-heavynode', 'so-node'] %}
+{%- set MANAGER = salt['grains.get']('host' '') %}
+{%- else %}
+{%- set MANAGER = salt['grains.get']('master') %}
+{%- endif %}
+
+output.elasticsearch:
+  enabled: true
+  hosts: ["https://{{ MANAGER }}:9200"]
+  ssl.certificate_authorities: ["/usr/share/filebeat/intraca.crt"]

salt/filebeat/etc/module_config.yml.jinja

@@ -0,0 +1,18 @@
+# DO NOT EDIT THIS FILE
+{%- if MODULES.modules is iterable and MODULES.modules is not string and MODULES.modules|length > 0%}
+  {%- for module in MODULES.modules.keys() %}
+- module: {{ module }}
+    {%- for fileset in MODULES.modules[module] %}
+  {{ fileset }}:
+    enabled: {{ MODULES.modules[module][fileset].enabled|string|lower }}
+      {#- only manage the settings if the fileset is enabled #}
+      {%- if MODULES.modules[module][fileset].enabled %}
+        {%- for var, value in MODULES.modules[module][fileset].items() %}
+          {%- if var|lower != 'enabled' %}
+    {{ var }}: {{ value }}
+          {%- endif %}
+        {%- endfor %}
+      {%- endif %}
+    {%- endfor %}
+  {%- endfor %}
+{% endif %}

salt/filebeat/init.sls

@@ -20,18 +20,37 @@
 {% set LOCALHOSTIP = salt['grains.get']('ip_interfaces').get(MAININT)[0] %}
 {% set MANAGER = salt['grains.get']('master') %}
 {% set MANAGERIP = salt['pillar.get']('global:managerip', '') %}
+{% from 'filebeat/map.jinja' import THIRDPARTY with context %}
+{% from 'filebeat/map.jinja' import SO with context %}
+{% set ES_INCLUDED_NODES = ['so-standalone', 'so-managersearch', 'so-node', 'so-heavynode', 'so-import'] %}
+
+#only include elastic state for certain nodes
+{% if grains.role in ES_INCLUDED_NODES %}
+include:
+  - elasticsearch
+{% endif %}
+
 filebeatetcdir:
   file.directory:
     - name: /opt/so/conf/filebeat/etc
     - user: 939
     - group: 939
     - makedirs: True
+
+filebeatmoduledir:
+  file.directory:
+    - name: /opt/so/conf/filebeat/modules
+    - user: root
+    - group: root
+    - makedirs: True
+
 filebeatlogdir:
   file.directory:
     - name: /opt/so/log/filebeat
     - user: 939
     - group: 939
     - makedirs: True
+
 filebeatpkidir:
   file.directory:
     - name: /opt/so/conf/filebeat/etc/pki
@@ -44,6 +63,7 @@ fileregistrydir:
     - user: 939
     - group: 939
     - makedirs: True
+
 # This needs to be owned by root
 filebeatconfsync:
   file.managed:
@@ -55,6 +75,32 @@ filebeatconfsync:
     - defaults:
         INPUTS: {{ salt['pillar.get']('filebeat:config:inputs', {}) }}
         OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }}
+
+# Filebeat module config file
+filebeatmoduleconfsync:
+  file.managed:
+    - name: /opt/so/conf/filebeat/etc/module-setup.yml
+    - source: salt://filebeat/etc/module-setup.yml
+    - user: root
+    - group: root
+    - template: jinja
+
+sodefaults_module_conf:
+  file.managed:
+    - name: /opt/so/conf/filebeat/modules/securityonion.yml
+    - source: salt://filebeat/etc/module_config.yml.jinja
+    - template: jinja
+    - defaults:
+        MODULES: {{ SO }}
+
+thirdparty_module_conf:
+  file.managed:
+    - name: /opt/so/conf/filebeat/modules/thirdparty.yml
+    - source: salt://filebeat/etc/module_config.yml.jinja
+    - template: jinja
+    - defaults:
+        MODULES: {{ THIRDPARTY }}
+    
 so-filebeat:
   docker_container.running:
     - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-filebeat:{{ VERSION }}
@@ -65,19 +111,40 @@ so-filebeat:
       - /nsm:/nsm:ro
       - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw
       - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+      - /opt/so/conf/filebeat/etc/module-setup.yml:/usr/share/filebeat/module-setup.yml:ro
       - /nsm/wazuh/logs/alerts:/wazuh/alerts:ro
       - /nsm/wazuh/logs/archives:/wazuh/archives:ro
+      - /opt/so/conf/filebeat/modules:/usr/share/filebeat/modules.d
       - /opt/so/conf/filebeat/etc/pki/filebeat.crt:/usr/share/filebeat/filebeat.crt:ro
       - /opt/so/conf/filebeat/etc/pki/filebeat.key:/usr/share/filebeat/filebeat.key:ro
       - /opt/so/conf/filebeat/registry:/usr/share/filebeat/data/registry:rw
       - /etc/ssl/certs/intca.crt:/usr/share/filebeat/intraca.crt:ro
+      - /opt/so/log:/logs:ro
     - port_bindings:
         - 0.0.0.0:514:514/udp
         - 0.0.0.0:514:514/tcp
         - 0.0.0.0:5066:5066/tcp
+{% for module in THIRDPARTY.modules.keys() %}
+  {% for submodule in THIRDPARTY.modules[module] %}
+    {% if THIRDPARTY.modules[module][submodule].enabled and THIRDPARTY.modules[module][submodule]["var.syslog_port"] is defined %}
+        - {{ THIRDPARTY.modules[module][submodule].get("var.syslog_host", "0.0.0.0") }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}:{{ THIRDPARTY.modules[module][submodule]["var.syslog_port"] }}/{{ THIRDPARTY.modules[module][submodule]["var.input"] }}
+    {% endif %}
+  {% endfor %}
+{% endfor %}
     - watch:
       - file: /opt/so/conf/filebeat/etc/filebeat.yml
 
+{% if grains.role in ES_INCLUDED_NODES %}
+run_module_setup:
+  cmd.run:
+    - name: /usr/sbin/so-filebeat-module-setup
+    - require:
+      - file: filebeatmoduleconfsync
+      - docker_container: so-filebeat
+    - onchanges:
+      - docker_container: so-elasticsearch
+{% endif %}
+
 append_so-filebeat_so-status.conf:
   file.append:
     - name: /opt/so/conf/so-status/so-status.conf

salt/filebeat/map.jinja

@@ -0,0 +1,6 @@
+{% import_yaml 'filebeat/thirdpartydefaults.yaml' as TPDEFAULTS %}
+{% set THIRDPARTY = salt['pillar.get']('filebeat:third_party_filebeat', default=TPDEFAULTS.third_party_filebeat, merge=True) %}
+
+{% import_yaml 'filebeat/securityoniondefaults.yaml' as SODEFAULTS %}
+{% set SO = SODEFAULTS.securityonion_filebeat %}
+{#% set SO = salt['pillar.get']('filebeat:third_party_filebeat', default=SODEFAULTS.third_party_filebeat, merge=True) %#}

salt/filebeat/securityoniondefaults.yaml

@@ -0,0 +1,31 @@
+{%- set ZEEKVER = salt['pillar.get']('global:mdengine', '') %}
+{% set ZEEKLOGLOOKUP = {
+    'conn': 'connection',
+} %}
+securityonion_filebeat:
+  modules:
+  {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone','so-node', 'so-hotnode', 'so-warmnode', 'so-heavynode']   %}
+    elasticsearch:
+      server:
+        enabled: true
+        var.paths: ["/logs/elasticsearch/*.log"]
+    logstash:
+      log:
+        enabled: true
+        var.paths: ["/logs/logstash.log"]
+  {%- endif %}
+  {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone'] %}
+    kibana:
+      log:
+        enabled: true
+        var.paths: ["/logs/kibana/kibana.log"]
+  {%- endif %}
+  {%- if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode'] %}
+    redis:
+      log:
+        enabled: true
+        var.paths: ["/logs/redis.log"]
+      slowlog:
+        enabled: false
+  {%- endif %}
+        

salt/filebeat/thirdpartydefaults.yaml

@@ -0,0 +1,259 @@
+third_party_filebeat:
+  modules:
+    aws:
+      cloudtrail:
+        enabled: false
+      cloudwatch:
+        enabled: false
+      ec2:
+        enabled: false
+      elb:
+        enabled: false
+      s3access:
+        enabled: false
+      vpcflow:
+        enabled: false
+    azure:
+      activitylogs:
+        enabled: false
+      platformlogs:
+        enabled: false
+      auditlogs:
+        enabled: false
+      signinlogs:
+        enabled: false
+    barracuda:
+      waf:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9503
+      spamfirewall:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9524
+    bluecoat:
+      director:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9505
+    cef:
+      log:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9003
+    checkpoint:
+      firewall:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9505
+    cisco:
+      asa:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9001
+      ftd:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9003
+      ios:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9002
+      nexus:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9506
+      meraki:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9525
+      umbrella:
+        enabled: false
+      amp:
+        enabled: false
+    cyberark:
+      corepas:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9527
+    cylance:
+      protect:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9508
+    f5:
+      bigipapm:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9504
+      bigipafm:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9528    
+    fortinet:
+      firewall:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9004
+      clientendpoint:
+        enabled: false
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9510
+      fortimail:
+        enabled: false
+        var.input: udp
+        var.syslog_port: 9350
+    gcp:
+      vpcflow:
+        enabled: false
+      firewall:
+        enabled: false
+      audit:
+        enabled: false
+    google_workspace:
+      saml:
+        enabled: false
+      user_accounts:
+        enabled: false
+      login:
+        enabled: false
+      admin:
+        enabled: false
+      drive:
+        enabled: false
+      groups:
+        enabled: false
+    imperva:
+      securesphere:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9511
+    infoblox:
+      nios:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9512
+    juniper:
+      junos:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9513
+      netscreen:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9523
+      srx:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9006
+    microsoft:
+      defender_atp:
+        enabled: false 
+      m365_defender:
+        enabled: false
+      dhcp:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9515
+    misp:
+      threat:
+        enabled: false
+    netflow:
+      log:
+        enabled: false
+        var.netflow_host: 0.0.0.0
+        var.netflow_port: 2055
+        var.internal_networks:
+          - private
+    netscout:
+      sightline:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9502
+    o365:
+      audit:
+        enabled: false
+    okta:
+      system:
+        enabled: false
+    proofpoint:
+      emailsecurity:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9531
+    radware:
+      defensepro:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9518
+    snort:
+      log:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9532
+    snyk:
+      audit:
+        enabled: false
+      vulnerabilities:
+        enabled: false
+    sonicwall:
+      firewall:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9519
+    sophos:
+      xg:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9005
+      utm:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9533
+    squid:
+      log:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9520
+    tomcat:
+      log:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9501
+    zscaler:
+      zia:
+        enabled: false 
+        var.input: udp
+        var.syslog_host: 0.0.0.0
+        var.syslog_port: 9521

salt/logstash/pipelines/config/so/0009_input_beats.conf

@@ -4,3 +4,8 @@ input {
     tags => [ "beat-ext" ]
   }
 }
+filter {
+  mutate {
+    rename => {"@metadata" => "metadata"}
+  }
+}

salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja

@@ -0,0 +1,20 @@
+{%- if grains['role'] == 'so-eval' -%}
+{%- set ES = salt['pillar.get']('manager:mainip', '') -%}
+{%- else %}
+{%- set ES = salt['pillar.get']('elasticsearch:mainip', '') -%}
+{%- endif %}
+output {
+ if [metadata][pipeline] {
+   elasticsearch {
+     id => "filebeat_modules_metadata_pipeline"
+     pipeline => "%{[metadata][pipeline]}"
+     hosts => "{{ ES }}"
+     index => "so-%{[event][module]}-%{+YYYY.MM.dd}"
+     template_name => "so-common"
+     template => "/templates/so-common-template.json"
+     template_overwrite => true
+     ssl => true
+     ssl_certificate_verification => false
+   }
+ }
+}

salt/pipeline/load.sls

@@ -0,0 +1,4 @@
+load_elastic_pipelines:
+  cmd.run: 
+    - name: /usr/sbin/so-filebeat-module-setup
+  

salt/top.sls

@@ -155,6 +155,7 @@ base:
     - domainstats
     {%- endif %}
     - docker_clean
+    - pipeline.load
 
   '*_manager and G@saltversion:{{saltversion}}':
     - match: compound
@@ -216,6 +217,7 @@ base:
     - domainstats
     {%- endif %}
     - docker_clean
+    - pipeline.load
 
   '*_standalone and G@saltversion:{{saltversion}}':
     - match: compound
@@ -289,6 +291,7 @@ base:
     - domainstats
     {%- endif %}
     - docker_clean
+    - pipeline.load
 
   '*_searchnode and G@saltversion:{{saltversion}}':
     - match: compound
@@ -318,6 +321,7 @@ base:
     {%- endif %}
     - schedule
     - docker_clean
+    - pipeline.load
 
   '*_managersearch and G@saltversion:{{saltversion}}':
     - match: compound
@@ -383,6 +387,7 @@ base:
     - domainstats
     {%- endif %}
     - docker_clean
+    - pipeline.load
 
   '*_heavynode and G@saltversion:{{saltversion}}':
     - match: compound
@@ -426,6 +431,7 @@ base:
     {%- endif %}
     - schedule
     - docker_clean
+    - pipeline.load
   
   '*_fleet and G@saltversion:{{saltversion}}':
     - match: compound
@@ -471,3 +477,4 @@ base:
     - zeek
     - schedule
     - docker_clean
+    - pipeline.load

salt/zeek/init.sls

@@ -76,9 +76,9 @@ zeekpolicysync:
 # Ensure the zeek spool tree (and state.db) ownership is correct
 zeekspoolownership:
   file.directory:
-    - name: /nsm/zeek/spool
+    - name: /nsm/zeek
     - user: 937
-    - max_depth: 0
+    - max_depth: 1
     - recurse:
       - user
 
@@ -183,6 +183,8 @@ so-zeek:
     - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-zeek:{{ VERSION }}
     - start: {{ START }}
     - privileged: True
+    - ulimits:
+      - core=0
     - binds:
       - /nsm/zeek/logs:/nsm/zeek/logs:rw
       - /nsm/zeek/spool:/nsm/zeek/spool:rw

setup/so-functions

@@ -2830,7 +2830,6 @@ zeek_logs_enabled() {
 			"    - conn"\
 			"    - dce_rpc"\
 			"    - dhcp"\
-			"    - dhcpv6"\
 			"    - dnp3"\
 			"    - dns"\
 			"    - dpd"\
@@ -2841,25 +2840,20 @@ zeek_logs_enabled() {
 			"    - irc"\
 			"    - kerberos"\
 			"    - modbus"\
-			"    - mqtt"\
 			"    - notice"\
 			"    - ntlm"\
-			"    - openvpn"\
 			"    - pe"\
 			"    - radius"\
 			"    - rfb"\
 			"    - rdp"\
-			"    - signatures"\
 			"    - sip"\
 			"    - smb_files"\
 			"    - smb_mapping"\
 			"    - smtp"\
 			"    - snmp"\
-			"    - software"\
 			"    - ssh"\
 			"    - ssl"\
 			"    - syslog"\
-			"    - telnet"\
 			"    - tunnel"\
 			"    - weird"\
 			"    - mysql"\
@@ -2871,7 +2865,6 @@ zeek_logs_enabled() {
 			"    - conn"\
 			"    - dce_rpc"\
 			"    - dhcp"\
-			"    - dhcpv6"\
 			"    - dnp3"\
 			"    - dns"\
 			"    - dpd"\
@@ -2882,24 +2875,19 @@ zeek_logs_enabled() {
 			"    - irc"\
 			"    - kerberos"\
 			"    - modbus"\
-			"    - mqtt"\
 			"    - notice"\
 			"    - ntlm"\
-			"    - openvpn"\
 			"    - pe"\
 			"    - radius"\
 			"    - rfb"\
 			"    - rdp"\
-			"    - signatures"\
 			"    - sip"\
 			"    - smb_files"\
 			"    - smb_mapping"\
 			"    - smtp"\
 			"    - snmp"\
-			"    - software"\
 			"    - ssh"\
 			"    - ssl"\
-			"    - telnet"\
 			"    - tunnel"\
 			"    - weird"\
 			"    - mysql"\

setup/so-whiptail

@@ -1155,7 +1155,6 @@ whiptail_manager_adv_service_zeeklogs() {
 	"conn" "Connection Logging" ON \
 	"dce_rpc" "RPC Logs" ON \
 	"dhcp" "DHCP Logs" ON \
-	"dhcpv6" "DHCP IPv6 Logs" ON \
 	"dnp3" "DNP3 Logs" ON \
 	"dns" "DNS Logs" ON \
 	"dpd" "DPD Logs" ON \
@@ -1166,25 +1165,20 @@ whiptail_manager_adv_service_zeeklogs() {
 	"irc" "IRC Chat Logs" ON \
 	"kerberos" "Kerberos Logs" ON \
 	"modbus" "MODBUS Logs" ON \
-	"mqtt" "MQTT Logs" ON \
 	"notice" "Zeek Notice Logs" ON \
 	"ntlm" "NTLM Logs" ON \
-	"openvpn" "OPENVPN Logs" ON \
 	"pe" "PE Logs" ON \
 	"radius" "Radius Logs" ON \
 	"rfb" "RFB Logs" ON \
 	"rdp" "RDP Logs" ON \
-	"signatures" "Signatures Logs" ON \
 	"sip" "SIP Logs" ON \
 	"smb_files" "SMB Files Logs" ON \
 	"smb_mapping" "SMB Mapping Logs" ON \
 	"smtp" "SMTP Logs" ON \
 	"snmp" "SNMP Logs" ON \
-	"software" "Software Logs" ON \
 	"ssh" "SSH Logs" ON \
 	"ssl" "SSL Logs" ON \
 	"syslog" "Syslog Logs" ON \
-	"telnet" "Telnet Logs" ON \
 	"tunnel" "Tunnel Logs" ON \
 	"weird" "Zeek Weird Logs" ON \
 	"mysql" "MySQL Logs" ON \