Merge branch 'dev' into feature/rotate-logs

# Conflicts:
#	setup/so-functions

salt/common/files/analyst/so-login-logo-dark.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 87.86 105.22"><defs><style>.cls-1{fill:#fff;}.cls-2{fill:#1976d2;}</style></defs><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><g id="Onion"><path id="Flesh" class="cls-1" d="M43.37,71.34a1.27,1.27,0,0,0,.44-.51,4.74,4.74,0,0,0,.61-2.39c-.12-6.79-.22-12.88-4-14.46-4.05-1.72-9.38,3.14-10.71,4.35a19.84,19.84,0,0,0-6.17,12.34c-.1,1-.76,9.34,5.46,15.41s15.45,6.06,21.72,3.53A22.25,22.25,0,0,0,61.88,79.16c5.31-10,1.61-20.31.85-22.3C57.78,44,43.35,36.11,29.88,36.78c-2.17.11-15.82,1-24.16,12.42A30.55,30.55,0,0,0,0,67.36c.15,16.14,13.38,29.51,26.23,34.7,12.61,5.1,24,2.76,28.78,1.65s17.12-4,25.53-15.08a34.47,34.47,0,0,0,7.24-18.46,34.79,34.79,0,0,0-3.42-17.32c-1.11-2.3-6.16-12.09-17-17C57,31.21,48.52,34.37,45.65,29.12a8.46,8.46,0,0,1-.41-6.21,1,1,0,0,0-1.05-1.28l-1.6,0a1.07,1.07,0,0,0-1,.8c-.66,2.51-1.12,6,.51,9.17C46,39.08,56.87,35.31,67.56,42.78c8.29,5.79,14.14,16.69,13.21,27.29a28.06,28.06,0,0,1-6,14.65c-7,9-17,11.29-21.82,12.38-4,.9-13.19,2.87-23.54-.93-2.65-1-20.33-8.29-22.38-25C5.72,60.55,13,48.9,24.21,44.93c13-4.6,27.26,2.75,32.09,13.26.58,1.25,4.85,10.93-.59,18.72-4.05,5.79-13.07,9.94-19.77,6A13.48,13.48,0,0,1,30,68.25c1.42-5,6.37-8.72,8.13-7.84s2.94,6.14,3,9.85A1.39,1.39,0,0,0,43.37,71.34Z"/><path id="Stem" class="cls-2" d="M30,27.14l-4.17,1.27a1.16,1.16,0,0,1-1.49-.93l-.11-.72a26.93,26.93,0,0,0-4.53-11.09A1.13,1.13,0,0,1,20.06,14l1.06-.63a1.15,1.15,0,0,1,1.52.32c.41.58.82,1.17,1.23,1.78l1.48,2.2C28.42,7.27,37.14.12,46.21,0,58.09-.16,65.59,10.67,68,17.63a23.37,23.37,0,0,1,.94,3.64.91.91,0,0,1-1.14,1l-2.66-.73a1.47,1.47,0,0,1-1-1.08,19.71,19.71,0,0,0-1.9-4.8c-3-5.44-9.67-11.21-16.55-10.59-7.74.7-15.22,9.46-14.85,20.91A1.14,1.14,0,0,1,30,27.14Z"/></g></g></g></svg>

salt/common/files/analyst/so-login-logo.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 87.86 105.22"><defs><style>.cls-1{fill:#1976d2;}</style></defs><g id="Layer_2" data-name="Layer 2"><g id="Layer_1-2" data-name="Layer 1"><g id="Onion"><path id="Flesh" d="M43.37,71.34a1.27,1.27,0,0,0,.44-.51,4.74,4.74,0,0,0,.61-2.39c-.12-6.79-.22-12.88-4-14.46-4.05-1.72-9.38,3.14-10.71,4.35a19.84,19.84,0,0,0-6.17,12.34c-.1,1-.76,9.34,5.46,15.41s15.45,6.06,21.72,3.53A22.25,22.25,0,0,0,61.88,79.16c5.31-10,1.61-20.31.85-22.3C57.78,44,43.35,36.11,29.88,36.78c-2.17.11-15.82,1-24.16,12.42A30.55,30.55,0,0,0,0,67.36c.15,16.14,13.38,29.51,26.23,34.7,12.61,5.1,24,2.76,28.78,1.65s17.12-4,25.53-15.08a34.47,34.47,0,0,0,7.24-18.46,34.79,34.79,0,0,0-3.42-17.32c-1.11-2.3-6.16-12.09-17-17C57,31.21,48.52,34.37,45.65,29.12a8.46,8.46,0,0,1-.41-6.21,1,1,0,0,0-1.05-1.28l-1.6,0a1.07,1.07,0,0,0-1,.8c-.66,2.51-1.12,6,.51,9.17C46,39.08,56.87,35.31,67.56,42.78c8.29,5.79,14.14,16.69,13.21,27.29a28.06,28.06,0,0,1-6,14.65c-7,9-17,11.29-21.82,12.38-4,.9-13.19,2.87-23.54-.93-2.65-1-20.33-8.29-22.38-25C5.72,60.55,13,48.9,24.21,44.93c13-4.6,27.26,2.75,32.09,13.26.58,1.25,4.85,10.93-.59,18.72-4.05,5.79-13.07,9.94-19.77,6A13.48,13.48,0,0,1,30,68.25c1.42-5,6.37-8.72,8.13-7.84s2.94,6.14,3,9.85A1.39,1.39,0,0,0,43.37,71.34Z"/><path id="Stem" class="cls-1" d="M30,27.14l-4.17,1.27a1.16,1.16,0,0,1-1.49-.93l-.11-.72a26.93,26.93,0,0,0-4.53-11.09A1.13,1.13,0,0,1,20.06,14l1.06-.63a1.15,1.15,0,0,1,1.52.32c.41.58.82,1.17,1.23,1.78l1.48,2.2C28.42,7.27,37.14.12,46.21,0,58.09-.16,65.59,10.67,68,17.63a23.37,23.37,0,0,1,.94,3.64.91.91,0,0,1-1.14,1l-2.66-.73a1.47,1.47,0,0,1-1-1.08,19.71,19.71,0,0,0-1.9-4.8c-3-5.44-9.67-11.21-16.55-10.59-7.74.7-15.22,9.46-14.85,20.91A1.14,1.14,0,0,1,30,27.14Z"/></g></g></g></svg>

salt/common/init.sls

@@ -56,6 +56,12 @@ salttmp:
 
 # Install epel
 {% if grains['os'] == 'CentOS' %}
+repair_yumdb:
+  cmd.run:
+    - name: 'mv -f /var/lib/rpm/__db* /tmp && yum clean all'
+    - onlyif:
+      - 'yum check-update 2>&1 | grep "Error: rpmdb open failed"'
+
 epel:
   pkg.installed:
     - skip_suggestions: True

salt/common/tools/sbin/so-analyst-install

@@ -0,0 +1,309 @@
+#!/bin/bash
+
+# Copyright 2014-2020 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+if [ "$(id -u)" -ne 0 ]; then
+	echo "This script must be run using sudo!"
+	exit 1
+fi
+
+INSTALL_LOG=/root/so-analyst-install.log
+exec &> >(tee -a "$INSTALL_LOG")
+
+log() {
+	msg=$1
+	level=${2:-I}
+	now=$(TZ=GMT date +"%Y-%m-%dT%H:%M:%SZ")
+	echo -e "$now | $level | $msg" >> "$INSTALL_LOG" 2>&1
+}
+
+error() {
+	log "$1" "E"
+}
+
+info() {
+	log "$1" "I"
+}
+
+title() {
+	echo -e "\n-----------------------------\n $1\n-----------------------------\n" >> "$INSTALL_LOG" 2>&1
+}
+
+logCmd() {
+	cmd=$1
+	info "Executing command: $cmd"
+	$cmd >> "$INSTALL_LOG" 2>&1
+}
+
+analyze_system() {
+	title "System Characteristics"
+	logCmd "uptime"
+	logCmd "uname -a"
+	logCmd "free -h"
+	logCmd "lscpu"
+	logCmd "df -h"
+	logCmd "ip a"
+}
+
+analyze_system
+
+OS=$(grep PRETTY_NAME /etc/os-release | grep 'CentOS Linux 7')
+if [ $? -ne 0 ]; then
+  echo "This is an unsupported OS. Please use CentOS 7 to install the analyst node."
+  exit 1
+fi
+
+if [[ "$manufacturer" == "Security Onion Solutions" && "$family" == "Automated" ]]; then
+  INSTALL=yes
+  CURLCONTINUE=no
+else
+  INSTALL=''
+  CURLCONTINUE=''
+fi
+
+FIRSTPASS=yes
+while [[ $INSTALL != "yes" ]] && [[ $INSTALL != "no" ]]; do
+  if [[ "$FIRSTPASS" == "yes" ]]; then
+    clear
+    echo "###########################################"
+    echo "##          ** W A R N I N G **          ##"
+    echo "##    _______________________________    ##"
+    echo "##                                       ##"
+    echo "##    Installing the Security Onion      ##"
+    echo "##   analyst node on this device will    ##"
+    echo "##      make permanenet changes to       ##"
+    echo "##              the system.              ##"
+    echo "##                                       ##"
+    echo "###########################################"
+    echo "Do you wish to continue? (Type the entire word 'yes' to proceed or 'no' to exit)"
+    FIRSTPASS=no
+  else
+    echo "Please type 'yes' to continue or 'no' to exit."
+  fi      
+  read INSTALL
+done
+
+if [[ $INSTALL == "no" ]]; then
+  echo "Exiting analyst node installation."
+  exit 0
+fi
+
+echo "Testing for internet connection with curl https://securityonionsolutions.com/"
+CANCURL=$(curl -sI https://securityonionsolutions.com/ | grep "200 OK")
+  if [ $? -ne 0 ]; then
+    FIRSTPASS=yes
+    while [[ $CURLCONTINUE != "yes" ]] && [[ $CURLCONTINUE != "no" ]]; do
+      if [[ "$FIRSTPASS" == "yes" ]]; then
+        echo "We could not access https://securityonionsolutions.com/."
+        echo "Since packages are downloaded from the internet, internet acceess is required."
+        echo "If you would like to ignore this warning and continue anyway, please type 'yes'."
+        echo "Otherwise, type 'no' to exit."
+        FIRSTPASS=no
+      else
+        echo "Please type 'yes' to continue or 'no' to exit."
+      fi  
+      read CURLCONTINUE
+    done
+    if [[ "$CURLCONTINUE" == "no" ]]; then
+      echo "Exiting analyst node installation."
+      exit 0
+    fi
+  else
+    echo "We were able to curl https://securityonionsolutions.com/."
+    sleep 3
+  fi
+
+# Install a GUI text editor
+yum -y install gedit
+
+# Install misc utils
+yum -y install wget curl unzip epel-release yum-plugin-versionlock;
+
+# Install xWindows
+yum -y groupinstall "X Window System";
+yum -y install gnome-classic-session gnome-terminal nautilus-open-terminal control-center liberation-mono-fonts;
+unlink /etc/systemd/system/default.target;
+ln -sf /lib/systemd/system/graphical.target /etc/systemd/system/default.target;
+yum -y install file-roller
+
+# Install Mono - prereq for NetworkMiner
+yum -y install mono-core mono-basic mono-winforms expect
+
+# Install NetworkMiner
+yum -y install libcanberra-gtk2;
+wget https://www.netresec.com/?download=NetworkMiner -O /tmp/nm.zip;
+mkdir -p /opt/networkminer/
+unzip /tmp/nm.zip -d /opt/networkminer/;
+rm /tmp/nm.zip;
+mv /opt/networkminer/NetworkMiner_*/* /opt/networkminer/
+chmod +x /opt/networkminer/NetworkMiner.exe;
+chmod -R go+w /opt/networkminer/AssembledFiles/;
+chmod -R go+w /opt/networkminer/Captures/;
+# Create networkminer shim
+cat << EOF >> /bin/networkminer
+#!/bin/bash
+/bin/mono /opt/networkminer/NetworkMiner.exe --noupdatecheck "\$@"
+EOF
+chmod +x /bin/networkminer
+# Convert networkminer ico file to png format
+yum -y install ImageMagick
+convert /opt/networkminer/networkminericon.ico /opt/networkminer/networkminericon.png
+# Create menu entry
+cat << EOF >> /usr/share/applications/networkminer.desktop
+[Desktop Entry]
+Name=NetworkMiner
+Comment=NetworkMiner
+Encoding=UTF-8
+Exec=/bin/networkminer %f
+Icon=/opt/networkminer/networkminericon-4.png
+StartupNotify=true
+Terminal=false
+X-MultipleArgs=false
+Type=Application
+MimeType=application/x-pcap;
+Categories=Network;
+EOF
+
+# Set default monospace font to Liberation
+cat << EOF >> /etc/fonts/local.conf
+ <match target="pattern">
+  <test name="family" qual="any">
+   <string>monospace</string>
+  </test>
+  <edit binding="strong" mode="prepend" name="family">
+   <string>Liberation Mono</string>
+  </edit>
+ </match>
+EOF
+
+# Install Wireshark for Gnome
+yum -y install wireshark-gnome; 
+
+# Install dnsiff
+yum -y install dsniff;
+
+# Install hping3
+yum -y install hping3;
+
+# Install netsed
+yum -y install netsed;
+
+# Install ngrep
+yum -y install ngrep;
+
+# Install scapy
+yum -y install python36-scapy;
+
+# Install ssldump
+yum -y install ssldump;
+
+# Install tcpdump
+yum -y install tcpdump;
+
+# Install tcpflow
+yum -y install tcpflow;
+
+# Install tcpxtract
+yum -y install tcpxtract;
+
+# Install whois
+yum -y install whois;
+
+# Install foremost
+yum -y install https://forensics.cert.org/centos/cert/7/x86_64//foremost-1.5.7-13.1.el7.x86_64.rpm;
+
+# Install chromium
+yum -y install chromium;
+
+# Install tcpstat
+yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-tcpstat-1.5.0/securityonion-tcpstat-1.5.0.rpm;
+
+# Install tcptrace
+yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-tcptrace-6.6.7/securityonion-tcptrace-6.6.7.rpm;
+
+# Install sslsplit
+yum -y install libevent;
+yum -y install sslsplit;
+
+# Install Bit-Twist
+yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-bittwist-2.0.0/securityonion-bittwist-2.0.0.rpm;
+
+# Install chaosreader
+yum -y install perl-IO-Compress perl-Net-DNS;
+yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-chaosreader-0.95.10/securityonion-chaosreader-0.95.10.rpm;
+chmod +x /bin/chaosreader;
+
+if [ -f ../../files/analyst/README ]; then
+  cp ../../files/analyst/README /;
+  cp ../../files/analyst/so-wallpaper.jpg /usr/share/backgrounds/;
+  cp ../../files/analyst/so-lockscreen.jpg /usr/share/backgrounds/;
+  cp ../../files/analyst/so-login-logo-dark.svg /usr/share/pixmaps/;
+else
+  cp /opt/so/saltstack/default/salt/common/files/analyst/README /;
+  cp /opt/so/saltstack/default/salt/common/files/analyst/so-wallpaper.jpg /usr/share/backgrounds/;
+  cp /opt/so/saltstack/default/salt/common/files/analyst/so-lockscreen.jpg /usr/share/backgrounds/;
+  cp /opt/so/saltstack/default/salt/common/files/analyst/so-login-logo-dark.svg /usr/share/pixmaps/;
+fi
+
+# Set background wallpaper
+cat << EOF >> /etc/dconf/db/local.d/00-background
+# Specify the dconf path
+[org/gnome/desktop/background]
+
+# Specify the path to the desktop background image file
+picture-uri='file:///usr/share/backgrounds/so-wallpaper.jpg'
+# Specify one of the rendering options for the background image:
+# 'none', 'wallpaper', 'centered', 'scaled', 'stretched', 'zoom', 'spanned'
+picture-options='zoom'
+# Specify the left or top color when drawing gradients or the solid color
+primary-color='000000'
+# Specify the right or bottom color when drawing gradients
+secondary-color='FFFFFF'
+EOF
+
+# Set lock screen
+cat << EOF >> /etc/dconf/db/local.d/00-screensaver
+[org/gnome/desktop/session]
+idle-delay=uint32 180
+
+[org/gnome/desktop/screensaver]
+lock-enabled=true
+lock-delay=uint32 120
+picture-options='zoom'
+picture-uri='file:///usr/share/backgrounds/so-lockscreen.jpg'
+EOF
+
+cat << EOF >> /etc/dconf/db/local.d/locks/screensaver
+/org/gnome/desktop/session/idle-delay
+/org/gnome/desktop/screensaver/lock-enabled
+/org/gnome/desktop/screensaver/lock-delay
+EOF
+
+# Do not show the user list at login screen
+cat << EOF >> /etc/dconf/db/local.d/00-login-screen
+[org/gnome/login-screen]
+logo='/usr/share/pixmaps/so-login-logo-dark.svg'
+disable-user-list=true
+EOF
+
+dconf update;
+
+echo
+echo "Analyst workstation has been installed!"
+echo "Press ENTER to reboot or Ctrl-C to cancel."
+read pause
+
+reboot;

salt/common/tools/sbin/so-elasticsearch-pipelines-list

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+. /usr/sbin/so-common
+if [ "$1" == "" ]; then
+    curl -s {{ NODEIP }}:9200/_ingest/pipeline/* | jq 'keys'
+else
+    curl -s {{ NODEIP }}:9200/_ingest/pipeline/$1 | jq
+fi

salt/common/tools/sbin/so-elasticsearch-templates-list

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+{%- set NODEIP = salt['pillar.get']('elasticsearch:mainip', '') -%}
+. /usr/sbin/so-common
+if [ "$1" == "" ]; then
+    curl -s {{ NODEIP }}:9200/_template/* | jq 'keys'
+else
+    curl -s {{ NODEIP }}:9200/_template/$1 | jq
+fi

salt/curator/files/bin/so-curator-closed-delete-delete

@@ -1,7 +1,7 @@
 
 #!/bin/bash
 
-{%- if grains['role'] in ['so-node', 'so-searchnode', 'so-heavynode'] %}
+{%- if grains['role'] in ['so-node', 'so-heavynode'] %}
   {%- set ELASTICSEARCH_HOST = salt['pillar.get']('elasticsearch:mainip', '') -%}  
   {%- set ELASTICSEARCH_PORT = salt['pillar.get']('elasticsearch:es_port', '') -%}
   {%- set LOG_SIZE_LIMIT = salt['pillar.get']('elasticsearch:log_size_limit', '') -%}

salt/elastalert/files/modules/so/playbook-es.py

@@ -16,7 +16,7 @@ class PlaybookESAlerter(Alerter):
             today = strftime("%Y.%m.%d", gmtime())
             timestamp = strftime("%Y-%m-%d"'T'"%H:%M:%S", gmtime())
             headers = {"Content-Type": "application/json"}
-            payload = {"rule.name": self.rule['play_title'],"rule.uuid": self.rule['play_id'],"event.severity": self.rule['event.severity'],"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"event.module": self.rule['event.module'],"event.dataset": self.rule['event.dataset'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event.severity_label": self.rule['sigma_level'],"rule.category": self.rule['rule.category'],"event_data": match, "@timestamp": timestamp}
+            payload = {"rule": { "name": self.rule['play_title'],"uuid": self.rule['play_id'],"category": self.rule['rule.category']},"event":{ "severity": self.rule['event.severity'],"module": self.rule['event.module'],"dataset": self.rule['event.dataset'],"severity_label": self.rule['sigma_level']},"kibana_pivot": self.rule['kibana_pivot'],"soc_pivot": self.rule['soc_pivot'],"play_url": self.rule['play_url'],"sigma_level": self.rule['sigma_level'],"event_data": match, "@timestamp": timestamp}
             url = f"http://{self.rule['elasticsearch_host']}/so-playbook-alerts-{today}/_doc/"
             requests.post(url, data=json.dumps(payload), headers=headers, verify=False)
                             

salt/elasticsearch/files/ingest/common

@@ -48,6 +48,7 @@
     { "rename":         { "field": "category",            "target_field": "event.category",              "ignore_failure": true, "ignore_missing": true  } },
     { "rename":         { "field": "message2.community_id", "target_field": "network.community_id",  "ignore_failure": true,  "ignore_missing": true  } },
     { "lowercase":         { "field": "event.dataset", "ignore_failure": true,  "ignore_missing": true  } },    
+    { "lowercase":         { "field": "network.transport", "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "destination.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "source.port", "type": "integer",  "ignore_failure": true,  "ignore_missing": true  } },
     { "convert":         { "field": "log.id.uid", "type": "string",  "ignore_failure": true,  "ignore_missing": true  } },

salt/elasticsearch/files/ingest/common.nids

@@ -0,0 +1,17 @@
+{
+  "description" : "common.nids",
+  "processors" : [
+    { "convert":        { "if": "ctx.rule.uuid != null",                "field": "rule.uuid",                   "type": "integer"                                                                               } },
+    { "set":            { "if": "ctx.rule?.uuid < 1000000",             "field": "rule.reference", "value": "https://www.snort.org/search?query={{rule.gid}}-{{rule.uuid}}"     } },
+    { "set":            { "if": "ctx.rule?.uuid > 1999999",             "field": "rule.reference", "value": "https://doc.emergingthreats.net/{{rule.uuid}}"                     } },
+    { "convert":        { "if": "ctx.rule.uuid != null",                        "field": "rule.uuid",                   "type": "string"                                                                               } },
+    { "dissect":        { "if": "ctx.rule.name != null",                        "field": "rule.name",                   "pattern" : "%{rule_type} %{rest_of_rulename} ",                                "ignore_failure": true  } },
+    { "set":            { "if": "ctx.rule_type == 'GPL'",       "field": "rule_ruleset", "value": "Snort GPL"                                                   } },
+    { "set":            { "if": "ctx.rule_type == 'ET'",        "field": "rule.ruleset", "value": "Emerging Threats"                                            } },
+    { "set":            { "if": "ctx.rule.severity == 3",   "field": "event.severity", "value": 1, "override": true }  },
+    { "set":            { "if": "ctx.rule.severity == 2",   "field": "event.severity", "value": 2, "override": true }  },
+    { "set":            { "if": "ctx.rule.severity == 1",   "field": "event.severity", "value": 3, "override": true }  },
+    { "remove":         { "field": ["rule_type", "rest_of_rulename"],     "ignore_failure": true                                                                  } },
+    { "pipeline":       { "name": "common"                                                                                                                      } }
+  ]
+}

salt/elasticsearch/files/ingest/common_nids

@@ -1,17 +0,0 @@
-{
-  "description" : "common_nids",
-  "processors" : [
-    { "convert": 	{ "field": "sid", 			"type": "integer" 										} },
-    { "set": 		{ "if": "ctx.sid < 1000000",		"field": "signature_info", "value": "https://www.snort.org/search?query={{gid}}-{{sid}}"	} },
-    { "set": 		{ "if": "ctx.sid > 1999999",		"field": "signature_info", "value": "https://doc.emergingthreats.net/{{sid}}"			} },
-    { "remove":		{ "if": "ctx.sid > 2999999",		"field": "signature_info" 									} },
-    { "set": 		{ "if": "ctx.priority == '1'",		"field": "severity", "value": "High"	 							} },
-    { "set": 		{ "if": "ctx.priority == '2'",		"field": "severity", "value": "Medium" 								} },
-    { "set": 		{ "if": "ctx.priority == '3'",		"field": "severity", "value": "Low" 								} },
-    { "dissect": 	{ "field": "alert",			"pattern" : "%{rule_type} %{category} ",				"ignore_failure": true 	} },
-    { "set": 		{ "if": "ctx.rule_type == 'GPL'",	"field": "rule_type", "value": "Snort GPL"							} },
-    { "set": 		{ "if": "ctx.rule_type == 'ET'",	"field": "rule_type", "value": "Emerging Threats"						} },
-    { "lowercase": 	{ "field": "category",												"ignore_failure": true	} },
-    { "pipeline":       { "name": "common"															} }
-  ]
-}

salt/elasticsearch/files/ingest/sguild_nids

@@ -1,25 +0,0 @@
-{
-  "description" : "sguild_nids",
-  "processors" : [
-    { 
-	"dissect": {
-		"field": "message",
-		"pattern" : "%{} %{} %{}  Alert Received: %{} %{priority} %{classification} %{interface} {%{alerttime}} %{} %{} {%{alert}} %{source_ip} %{destination_ip} %{protocol} %{source_port} %{destination_port} %{gid} %{sid} %{rev} ",
-		"on_failure": [ { "drop" : { } } ]
-	}
-    },
-    { "set":		{ "if": "ctx.protocol == '1'",		"field": "protocol", "value": "ICMP"							} },
-    { "set":		{ "if": "ctx.protocol == '6'",		"field": "protocol", "value": "TCP" 							} },
-    { "set":		{ "if": "ctx.protocol == '17'",		"field": "protocol", "value": "UDP" 							} },
-    { "remove":		{ "if": "ctx.source_ip == '{}'",  	"field": "source_ip"									} },
-    { "remove":		{ "if": "ctx.destination_ip == '{}'", 	"field": "destination_ip"								} },
-    { "remove":		{ "if": "ctx.protocol == '{}'", 	"field": "protocol"									} },
-    { "remove":		{ "if": "ctx.source_port == '{}'",  	"field": "source_port"									} },
-    { "remove":		{ "if": "ctx.destination_port == '{}'", "field": "destination_port"								} },
-    { "set":		{ "field": "type", "value": "snort" 												} },
-    { "rename":		{ "field": "@timestamp",	"target_field": "timestamp",						"ignore_missing": true  } },
-    { "date":		{ "field": "alerttime",		"target_field": "@timestamp",	"formats": ["yyyy-MM-dd HH:mm:ss"],	"ignore_failure": true  } },
-    { "remove":		{ "field": "alerttime",											"ignore_missing": true  } },
-    { "pipeline": 	{ "name": "common_nids" 													} }
-  ]
-}

salt/elasticsearch/files/ingest/snort

@@ -1,21 +0,0 @@
-{
-  "description" : "snort",
-  "processors" : [
-    { 
-	"dissect": {
-		"field": "message",
-		"pattern" : "[%{gid}:%{sid}:%{rev}] %{alert} [Classification: %{classification}] [Priority: %{priority}]: <%{interface}> {%{protocol}} %{source_ip_port} -> %{destination_ip_port}",
-		"on_failure": [ { "drop" : { } } ]
-	}
-    },
-    { "split": { "field": "source_ip_port", 		"separator": ":", 			"ignore_failure": true 	} },
-    { "split": { "field": "destination_ip_port", 	"separator": ":", 			"ignore_failure": true 	} },
-    { "rename":{ "field": "source_ip_port.1",		"target_field": "source_port",		"ignore_failure": true 	} },
-    { "rename":{ "field": "destination_ip_port.1",	"target_field": "destination_port",	"ignore_failure": true 	} },
-    { "rename":{ "field": "source_ip_port.0",		"target_field": "source_ip",		"ignore_failure": true 	} },
-    { "rename":{ "field": "destination_ip_port.0",	"target_field": "destination_ip",	"ignore_failure": true 	} },
-    { "remove":{ "field": "source_ip_port", 							"ignore_failure": true	} },
-    { "remove":{ "field": "destination_ip_port", 						"ignore_failure": true	} },
-    { "pipeline": { "name": "common_nids" } }
-  ]
-}

salt/elasticsearch/files/ingest/strelka.file

@@ -6,7 +6,8 @@
     { "rename":         { "field": "message2.scan",        "target_field": "scan",          "ignore_missing": true  } },
     { "rename":         { "field": "message2.request",        "target_field": "request",          "ignore_missing": true  } },
     { "rename":         { "field": "scan.hash",        "target_field": "hash",          "ignore_missing": true  } },
-    { "grok":           { "field": "request.attributes.filename",       "patterns": ["-%{WORD:log.id.fuid}-"] } },
+
+    { "grok":           { "if": "ctx.request?.attributes?.filename != null",   "field": "request.attributes.filename",       "patterns": ["-%{WORD:log.id.fuid}-"], "ignore_failure": true } },
     { "foreach":
       {
         "if": "ctx.scan?.exiftool?.keys !=null",
@@ -19,8 +20,29 @@
           }
       }
     },
+    { "foreach":
+      {
+        "if": "ctx.scan?.yara?.meta !=null",
+          "field": "scan.yara.meta",
+          "processor":{
+            "set": {
+              "field": "rule.{{_ingest._value.identifier}}",
+              "value": "{{_ingest._value.value}}"
+            }
+          }
+      }
+    },
+    { "set":         { "if": "ctx.scan?.yara?.matches != null", "field": "rule.name",        "value": "{{scan.yara.matches.0}}" }},
+    { "set":         { "if": "ctx.scan?.yara?.matches != null", "field": "dataset",        "value": "alert", "override": true }},
+    { "rename":         { "field": "scan.hash",        "target_field": "",          "ignore_missing": true  } },
+    { "set":         { "if": "ctx.rule?.name != null && ctx.rule?.score == null",   "field": "event.severity", "value": 3, "override": true }  },
+    { "convert" :    { "if": "ctx.rule?.score != null", "field" : "rule.score","type": "integer"}},
+    { "set":         { "if": "ctx.rule?.score != null && ctx.rule?.score >= 0 && ctx.rule?.score <= 49",   "field": "event.severity", "value": 1, "override": true }  },
+    { "set":         { "if": "ctx.rule?.score != null && ctx.rule?.score >= 50 && ctx.rule?.score <=69",   "field": "event.severity", "value": 2, "override": true }  },
+    { "set":         { "if": "ctx.rule?.score != null && ctx.rule?.score >= 70 && ctx.rule?.score <=89",   "field": "event.severity", "value": 3, "override": true }  },
+    { "set":         { "if": "ctx.rule?.score != null && ctx.rule?.score >= 90",   "field": "event.severity", "value": 4, "override": true }  },
     { "set":         { "field": "observer.name",        "value": "{{agent.name}}" }},
-    { "remove":         { "field": ["host", "path", "message", "scan.exiftool.keys"],                                         "ignore_missing": true  } },
+    { "remove":         { "field": ["host", "path", "message", "scan.exiftool.keys", "scan.yara.meta"],                                         "ignore_missing": true  } },
     { "pipeline":       { "name": "common"                                                                                   } }
   ]
 }

salt/elasticsearch/files/ingest/suricata.alert

@@ -7,9 +7,6 @@
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.uuid", "ignore_failure": true  } },
     { "rename":{ "field": "rule.signature_id",     "target_field": "rule.signature", "ignore_failure": true  } },
     { "rename":{ "field": "message2.payload_printable",     "target_field": "network.data.decoded", "ignore_failure": true  } },
-    { "set":         { "if": "ctx.rule.severity == 3",   "field": "event.severity", "value": 1, "override": true }  },
+    { "pipeline": { "name": "common.nids" } }
-    { "set":         { "if": "ctx.rule.severity == 2",   "field": "event.severity", "value": 2, "override": true }  },
-    { "set":         { "if": "ctx.rule.severity == 1",   "field": "event.severity", "value": 3, "override": true }  },
-    { "pipeline": { "name": "common" } }
   ]
 }

salt/elasticsearch/files/ingest/zeek.tunnels

@@ -3,15 +3,6 @@
   "processors" : [
     { "remove":         { "field": ["host"],     "ignore_failure": true                                                                  } },
     { "json":		{ "field": "message",			"target_field": "message2",		"ignore_failure": true	} },
-    { "rename": 	{ "field": "message2.uid", 	 	"target_field": "log.id.uid",			"ignore_missing": true 	} },
-    { "dot_expander": 	{ "field": "id.orig_h", 		"path": "message2", 			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_h", 	"target_field": "source.ip",		"ignore_missing": true 	} },
-    { "dot_expander": 	{ "field": "id.orig_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.orig_p", 	"target_field": "source.port",		"ignore_missing": true 	} },
-    { "dot_expander": 	{ "field": "id.resp_h", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_h", 	"target_field": "destination.ip",	"ignore_missing": true 	} },
-    { "dot_expander": 	{ "field": "id.resp_p", 		"path": "message2",			"ignore_failure": true 	} },
-    { "rename": 	{ "field": "message2.id.resp_p", 	"target_field": "destination.port",	"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.tunnel_type", 	"target_field": "tunnel.type",		"ignore_missing": true 	} },
     { "rename": 	{ "field": "message2.action", 		"target_field": "event.action",		"ignore_missing": true 	} },
     { "pipeline":       { "name": "zeek.common"                                                                                   } }

salt/elasticsearch/init.sls

@@ -237,10 +237,10 @@ so-elasticsearch-pipelines:
       - file: esyml
       - file: so-elasticsearch-pipelines-file
 
-{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-searchnode', 'so-import'] and TEMPLATES %}
+{% if grains['role'] in ['so-manager', 'so-eval', 'so-managersearch', 'so-standalone', 'so-heavynode', 'so-node', 'so-import'] and TEMPLATES %}
 so-elasticsearch-templates:
   cmd.run:
-    - name: /usr/sbin/so-elasticsearch-templates
+    - name: /usr/sbin/so-elasticsearch-templates-load
     - cwd: /opt/so
 {% endif %}
 

salt/elasticsearch/templates/so/so-common-template.json

@@ -353,6 +353,9 @@
           "type":"object",
           "dynamic":true,
           "properties":{
+            "score":{
+              "type":"long"
+            },
             "uuid":{
               "type":"keyword"
             }

salt/grafana/dashboards/standalone/standalone.json

@@ -4231,6 +4231,139 @@
         "alignLevel": null
       }
     },
+    {
+      "aliasColors": {},
+      "bars": true,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "InfluxDB",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 5,
+        "w": 4,
+        "x": 16,
+        "y": 25
+      },
+      "hiddenSeries": false,
+      "id": 71,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": false,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "dataLinks": []
+      },
+      "percentage": false,
+      "pointradius": 2,
+      "points": true,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "groupBy": [
+            {
+              "params": [
+                "$__interval"
+              ],
+              "type": "time"
+            },
+            {
+              "params": [
+                "null"
+              ],
+              "type": "fill"
+            }
+          ],
+          "measurement": "zeekcaptureloss",
+          "orderByTime": "ASC",
+          "policy": "autogen",
+          "refId": "A",
+          "resultFormat": "time_series",
+          "select": [
+            [
+              {
+                "params": [
+                  "loss"
+                ],
+                "type": "field"
+              },
+              {
+                "params": [],
+                "type": "mean"
+              }
+            ]
+          ],
+          "tags": [
+            {
+              "key": "host",
+              "operator": "=",
+              "value": "{{ SERVERNAME }}"
+            }
+          ]
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "{{ SERVERNAME }} - Zeek Capture Loss",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:198",
+          "decimals": 1,
+          "format": "percent",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:199",
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },    
     {
       "aliasColors": {},
       "bars": false,
@@ -4428,7 +4561,7 @@
               "type": "fill"
             }
           ],
-          "measurement": "brodrop",
+          "measurement": "zeekdrop",
           "orderByTime": "ASC",
           "policy": "default",
           "refId": "A",
@@ -4867,8 +5000,8 @@
       "fillGradient": 0,
       "gridPos": {
         "h": 5,
-        "w": 8,
+        "w": 4,
-        "x": 16,
+        "x": 20,
         "y": 30
       },
       "hiddenSeries": false,

salt/kibana/files/saved_objects.ndjson

@@ -59,7 +59,7 @@
 {"attributes":{"columns":["osquery.hostname","osquery.columns.username","osquery.LiveQuery","osquery.columns.name"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"event_type: osquery AND osquery.name:*chrome*\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"title":"osquery - Chrome Extensions","version":1},"id":"0eee4360-18d4-11e9-932c-d12d2cf4ee95","migrationVersion":{"search":"7.4.0"},"references":[{"id":"*:logstash-*","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-09-16T11:45:16.810Z","version":"WzU3LDFd"}
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"},"savedSearchRefName":"search_0","title":"osquery - Chrome Extensions - Changes by Hostname","uiStateJSON":"{\n  \"vis\": {\n    \"params\": {\n      \"sort\": {\n        \"columnIndex\": null,\n        \"direction\": null\n      }\n    }\n  }\n}","version":1,"visState":"{\"title\":\"osquery - Chrome Extensions - Changes by Hostname\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Changes\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"osquery.hostname.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Hostname\"}}]}"},"id":"04f86530-1a59-11e9-ac0b-cb0ba10141ab","migrationVersion":{"visualization":"7.8.0"},"references":[{"id":"0eee4360-18d4-11e9-932c-d12d2cf4ee95","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-09-16T11:45:16.810Z","version":"WzU4LDFd"}
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"network.protocol:*\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Security Onion - Top Network Protocols","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Security Onion - Top Network Protocols\",\"type\":\"tagcloud\",\"params\":{\"scale\":\"linear\",\"orientation\":\"single\",\"minFontSize\":18,\"maxFontSize\":72,\"showLabel\":false,\"metric\":{\"type\":\"vis_dimension\",\"accessor\":1,\"format\":{\"id\":\"string\",\"params\":{}}},\"bucket\":{\"type\":\"vis_dimension\",\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"network.protocol.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Protocol\"}}]}"},"id":"d9eb5b30-6ea9-11ea-9266-1fd14ca6af34","migrationVersion":{"visualization":"7.8.0"},"references":[{"id":"2289a0c0-6970-11ea-a0cd-ffa0f6a1bc29","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-09-16T11:45:16.810Z","version":"WzU5LDFd"}
-{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{}"},"savedSearchRefName":"search_0","title":"Security Onion - Network - Transport","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Security Onion - Network - Transport\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":false,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"network.transport: Descending\",\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"network.transport\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"79cc9670-6ead-11ea-9266-1fd14ca6af34","migrationVersion":{"visualization":"7.8.0"},"references":[{"id":"387f44c0-6ea7-11ea-9266-1fd14ca6af34","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-09-16T11:45:16.810Z","version":"WzYwLDFd"}
+{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{}"},"savedSearchRefName":"search_0","title":"Security Onion - Network - Transport","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Security Onion - Network - Transport\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":false,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100},\"dimensions\":{\"metric\":{\"accessor\":1,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"},\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://PLACEHOLDER\",\"pathname\":\"/kibana/app/kibana\",\"basePath\":\"/kibana\"}}},\"params\":{},\"label\":\"network.transport: Descending\",\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"network.transport.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}"},"id":"79cc9670-6ead-11ea-9266-1fd14ca6af34","migrationVersion":{"visualization":"7.8.0"},"references":[{"id":"387f44c0-6ea7-11ea-9266-1fd14ca6af34","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-09-16T11:45:16.810Z","version":"WzYwLDFd"}
 {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"event.category: network\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.9.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":13,\"h\":9,\"i\":\"b18f064d-2fba-45d8-a3c3-700ecec939a3\"},\"panelIndex\":\"b18f064d-2fba-45d8-a3c3-700ecec939a3\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"7.9.0\",\"gridData\":{\"x\":13,\"y\":0,\"w\":14,\"h\":9,\"i\":\"3bf59d17-132e-47bc-b548-e1e073491ec5\"},\"panelIndex\":\"3bf59d17-132e-47bc-b548-e1e073491ec5\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"7.9.0\",\"gridData\":{\"x\":27,\"y\":0,\"w\":21,\"h\":9,\"i\":\"49c9ae10-3f16-4cec-b044-c5cf2db199ae\"},\"panelIndex\":\"49c9ae10-3f16-4cec-b044-c5cf2db199ae\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.9.0\",\"gridData\":{\"x\":0,\"y\":9,\"w\":13,\"h\":19,\"i\":\"87427ca5-e0b9-4ec8-bb5f-3452803befe1\"},\"panelIndex\":\"87427ca5-e0b9-4ec8-bb5f-3452803befe1\",\"embeddableConfig\":{\"legendOpen\":false,\"vis\":{\"legendOpen\":true}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.9.0\",\"gridData\":{\"x\":13,\"y\":9,\"w\":11,\"h\":19,\"i\":\"3d3199e1-d839-4738-bc99-e030365b7070\"},\"panelIndex\":\"3d3199e1-d839-4738-bc99-e030365b7070\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"},{\"version\":\"7.9.0\",\"gridData\":{\"x\":24,\"y\":9,\"w\":12,\"h\":19,\"i\":\"a7745b0f-1c69-4837-9f7e-3d79b5a2ac60\"},\"panelIndex\":\"a7745b0f-1c69-4837-9f7e-3d79b5a2ac60\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"7.9.0\",\"gridData\":{\"x\":36,\"y\":9,\"w\":12,\"h\":19,\"i\":\"221a543a-98d4-46dd-8e7c-bf97bb292021\"},\"panelIndex\":\"221a543a-98d4-46dd-8e7c-bf97bb292021\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"7.9.0\",\"gridData\":{\"x\":0,\"y\":28,\"w\":48,\"h\":29,\"i\":\"55902091-6959-4127-a969-4015fbf124d3\"},\"panelIndex\":\"55902091-6959-4127-a969-4015fbf124d3\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"}]","timeRestore":false,"title":"Security Onion - Network","version":1},"id":"04ff3ef0-6ea4-11ea-9266-1fd14ca6af34","migrationVersion":{"dashboard":"7.3.0"},"references":[{"id":"7f822930-6ea4-11ea-9266-1fd14ca6af34","name":"panel_0","type":"visualization"},{"id":"d04b5130-6e99-11ea-9266-1fd14ca6af34","name":"panel_1","type":"visualization"},{"id":"d9eb5b30-6ea9-11ea-9266-1fd14ca6af34","name":"panel_2","type":"visualization"},{"id":"79cc9670-6ead-11ea-9266-1fd14ca6af34","name":"panel_3","type":"visualization"},{"id":"ad398b70-6e9a-11ea-9266-1fd14ca6af34","name":"panel_4","type":"visualization"},{"id":"6b18be30-72a7-11ea-8dd2-9d8795a1200b","name":"panel_5","type":"visualization"},{"id":"b6a4f3f0-72a7-11ea-8dd2-9d8795a1200b","name":"panel_6","type":"visualization"},{"id":"8b6f3150-72a2-11ea-8dd2-9d8795a1200b","name":"panel_7","type":"search"}],"type":"dashboard","updated_at":"2020-09-16T14:28:35.705Z","version":"WzIyNjcsMl0="}
 {"attributes":{"columns":["source_ip","source_port","destination_ip","destination_port","uid","_id"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"filter\":[],\"size\":10,\"query\":{\"query_string\":{\"query\":\"event_type:bro_conn\",\"analyze_wildcard\":true}},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"title":"Connections - Logs","version":1},"id":"9bf42190-342d-11e7-9e93-53b62e1857b2","migrationVersion":{"search":"7.4.0"},"references":[{"id":"*:logstash-*","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2020-09-16T11:45:16.810Z","version":"WzYyLDFd"}
 {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[]}"},"savedSearchRefName":"search_0","title":"Connections - Missed Bytes","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Connections - Missed Bytes\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"missed_bytes\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"Missed Bytes\"}}],\"listeners\":{}}"},"id":"05088150-3670-11e7-8c78-e3086faf385c","migrationVersion":{"visualization":"7.8.0"},"references":[{"id":"9bf42190-342d-11e7-9e93-53b62e1857b2","name":"search_0","type":"search"}],"type":"visualization","updated_at":"2020-09-16T11:45:16.810Z","version":"WzYzLDFd"}

salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja

@@ -1,5 +1,6 @@
 {%- set MANAGER = salt['grains.get']('master') %}
 {%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %}
+{% set BATCH = salt['pillar.get']('logstash_settings:ls_pipeline_batch_size', 125) %}
 
 input {
 	redis {
@@ -10,5 +11,6 @@ input {
 		key => 'logstash:unparsed'
 		type => 'redis-input'
 		threads => {{ THREADS }}
+		batch_count => {{ BATCH }}
 	}
 }

salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja

@@ -7,7 +7,7 @@
 output {
   if [module] =~ "ossec" {
     elasticsearch {
-      pipeline => "%{module}.%{dataset}"
+      pipeline => "%{module}"
       hosts => "{{ ES }}"
       index => "so-ossec-%{+YYYY.MM.dd}"
       template_name => "so-ossec"

salt/soc/files/soc/alerts.queries.json

@@ -1,9 +1,9 @@
 [
-  { "name": "Group By Name, Module", "query": "* | groupby rule.name rule.uuid event.module event.severity_label" },
+  { "name": "Group By Name, Module", "query": "* | groupby rule.name event.module event.severity_label" },
-  { "name": "Group By Sensor, Source IP/Port, Destination IP/Port, Name", "query": "* | groupby observer.name source.ip source.port destination.ip destination.port rule.name rule.uuid network.community_id event.severity_label" },
+  { "name": "Group By Sensor, Source IP/Port, Destination IP/Port, Name", "query": "* | groupby observer.name source.ip source.port destination.ip destination.port rule.name network.community_id event.severity_label" },
-  { "name": "Group By Source IP, Name", "query": "* | groupby source.ip rule.name rule.uuid event.severity_label" },
+  { "name": "Group By Source IP, Name", "query": "* | groupby source.ip rule.name event.severity_label" },
-  { "name": "Group By Source Port, Name", "query": "* | groupby source.port rule.name rule.uuid event.severity_label" },
+  { "name": "Group By Source Port, Name", "query": "* | groupby source.port rule.name event.severity_label" },
-  { "name": "Group By Destination IP, Name", "query": "* | groupby destination.ip rule.name rule.uuid event.severity_label" },
+  { "name": "Group By Destination IP, Name", "query": "* | groupby destination.ip rule.name event.severity_label" },
-  { "name": "Group By Destination Port, Name", "query": "* | groupby destination.port rule.name rule.uuid event.severity_label" },
+  { "name": "Group By Destination Port, Name", "query": "* | groupby destination.port rule.name event.severity_label" },
   { "name": "Ungroup", "query": "*" }
-]
+]

salt/soc/files/soc/hunt.queries.json

@@ -28,9 +28,8 @@
           { "name": "DPD", "description": "Dynamic Protocol Detection errors", "query": "event.dataset:dpd | groupby error.reason"},
           { "name": "Files", "description": "Files grouped by mimetype", "query": "event.dataset:file | groupby file.mime_type source.ip"},
           { "name": "Files", "description": "Files grouped by source", "query": "event.dataset:file | groupby file.source source.ip"},
-          { "name": "FTP", "description": "FTP grouped by argument", "query": "event.dataset:ftp | groupby ftp.argument"},
+          { "name": "FTP", "description": "FTP grouped by command and argument", "query": "event.dataset:ftp | groupby ftp.command ftp.argument"},
-          { "name": "FTP", "description": "FTP grouped by command", "query": "event.dataset:ftp | groupby ftp.command"},
+          { "name": "FTP", "description": "FTP grouped by username and argument", "query": "event.dataset:ftp | groupby ftp.user ftp.argument"},
-          { "name": "FTP", "description": "FTP grouped by username", "query": "event.dataset:ftp | groupby ftp.user"},
           { "name": "HTTP", "description": "HTTP grouped by destination port", "query": "event.dataset:http | groupby destination.port"},
           { "name": "HTTP", "description": "HTTP grouped by status code and message", "query": "event.dataset:http | groupby http.status_code http.status_message"},
           { "name": "HTTP", "description": "HTTP grouped by method and user agent", "query": "event.dataset:http | groupby http.method http.useragent"},

salt/ssl/init.sls

@@ -390,7 +390,7 @@ fleetkeyperms:
     - group: 939
 
 {% endif %}
-{% if grains['role'] in ['so-sensor', 'so-manager', 'so-node', 'so-searchnode', 'so-eval', 'so-helix', 'so-managersearch', 'so-heavynode', 'so-fleet', 'so-standalone', 'so-import'] %}
+{% if grains['role'] in ['so-sensor', 'so-manager', 'so-node', 'so-eval', 'so-helix', 'so-managersearch', 'so-heavynode', 'so-fleet', 'so-standalone', 'so-import'] %}
    
 fbcertdir:
   file.directory:
@@ -527,7 +527,7 @@ fleetkeyperms:
 
 {% endif %}
 
-{% if grains['role'] in ['so-node', 'so-searchnode'] %}
+{% if grains['role'] == 'so-node' %}
 # Create a cert for elasticsearch
 /etc/pki/elasticsearch.key:
   x509.private_key_managed:

salt/suricata/afpacket.map.jinja

@@ -5,6 +5,7 @@ af-packet:
     cluster-type: cluster_flow
     defrag: yes
     use-mmap: yes
+    threads: {{ salt['pillar.get']('sensor:suriprocs', salt['pillar.get']('sensor:suripins') | length) }}
     tpacket-v3: yes
     ring-size: {{ salt['pillar.get']('sensor:suriringsize', '2048') }}
   - interface: default

salt/suricata/defaults.yaml

@@ -345,7 +345,7 @@ suricata:
     coredump:
       max-dump: unlimited
     host-mode: auto
-    max-pending-packets: 1024
+    max-pending-packets: 5000
     runmode: workers
     #autofp-scheduler: hash
     default-packet-size: 1500

salt/telegraf/etc/telegraf.conf

@@ -679,7 +679,8 @@
      "/scripts/stenoloss.sh",
      "/scripts/suriloss.sh",
      "/scripts/checkfiles.sh",
-     "/scripts/broloss.sh",
+     "/scripts/zeekloss.sh",
+     "/scripts/zeekcaptureloss.sh",
      "/scripts/oldpcap.sh"
    ]
    data_format = "influx"
@@ -691,7 +692,8 @@
      "/scripts/stenoloss.sh",
      "/scripts/suriloss.sh",
      "/scripts/checkfiles.sh",
-     "/scripts/broloss.sh",
+     "/scripts/zeekloss.sh",
+     "/scripts/zeekcaptureloss.sh",
      "/scripts/oldpcap.sh"
    ]
    data_format = "influx"
@@ -702,7 +704,8 @@
      "/scripts/stenoloss.sh",
      "/scripts/suriloss.sh",
      "/scripts/checkfiles.sh",
-     "/scripts/broloss.sh",
+     "/scripts/zeekloss.sh",
+     "/scripts/zeekcaptureloss.sh",
      "/scripts/oldpcap.sh",
      "/scripts/influxdbsize.sh"
    ]
@@ -713,7 +716,8 @@
     "/scripts/stenoloss.sh",
     "/scripts/suriloss.sh",
     "/scripts/checkfiles.sh",
-    "/scripts/broloss.sh",
+    "/scripts/zeekloss.sh",
+    "/scripts/zeekcaptureloss.sh",
     "/scripts/oldpcap.sh",
     "/scripts/helixeps.sh"
   ]

salt/telegraf/scripts/zeekcaptureloss.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# This script returns the average of all the workers average capture loss to telegraf / influxdb in influx format include nanosecond precision timestamp
+{%- set WORKERS = salt['pillar.get']('sensor:zeek_lbprocs', salt['pillar.get']('sensor:zeek_pins') | length) %}
+ZEEKLOG=/host/nsm/zeek/spool/logger/capture_loss.log
+LASTCAPTURELOSSLOG=/var/log/telegraf/lastcaptureloss.txt
+if [ -f "$ZEEKLOG" ]; then
+  CURRENTTS=$(tail -1 $ZEEKLOG | jq .ts | sed 's/"//g')
+  if [ -f "$LASTCAPTURELOSSLOG" ]; then
+    LASTTS=$(cat $LASTCAPTURELOSSLOG)
+    if [[ "$LASTTS" != "$CURRENTTS" ]]; then
+      LOSS=$(tail -{{WORKERS}} $ZEEKLOG | awk -F, '{print $NF}' | sed 's/}//' | awk -F: '{LOSS += $2 / {{WORKERS}}} END { print LOSS}')
+      echo "zeekcaptureloss loss=$LOSS"
+    fi
+  fi
+  echo "$CURRENTTS" > $LASTCAPTURELOSSLOG
+fi

salt/telegraf/scripts/zeekloss.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-
+# This script returns the packets dropped by Zeek, but it isn't a percentage. $LOSS * 100 would be the percentage
 ZEEKLOG=$(tac /host/nsm/zeek/logs/packetloss.log | head -2)
 declare RESULT=($ZEEKLOG)
 CURRENTDROP=${RESULT[3]}
@@ -7,11 +7,11 @@ PASTDROP=${RESULT[9]}
 DROPPED=$((CURRENTDROP - PASTDROP))
 if [ $DROPPED == 0 ]; then
   LOSS=0
-  echo "brodrop drop=0"
+  echo "zeekdrop drop=0"
 else
   CURRENTPACKETS=${RESULT[5]}
   PASTPACKETS=${RESULT[11]}
   TOTAL=$((CURRENTPACKETS - PASTPACKETS))
   LOSS=$(echo $DROPPED $TOTAL / p | dc)
-  echo "brodrop drop=$LOSS"
+  echo "zeekdrop drop=$LOSS"
 fi

setup/so-analyst

@@ -1,177 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014-2020 Security Onion Solutions, LLC
-
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License as published by
-#    the Free Software Foundation, either version 3 of the License, or
-#    (at your option) any later version.
-#
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
-#
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-if [ "$(id -u)" -ne 0 ]; then
-	echo "This script must be run using sudo!"
-	exit 1
-fi
-
-# Install a GUI text editor
-yum -y install gedit
-
-# Install misc utils
-yum -y install wget curl unzip epel-release;
-
-# Install xWindows
-yum -y groupinstall "X Window System";
-yum -y install gnome-classic-session gnome-terminal nautilus-open-terminal control-center liberation-mono-fonts;
-unlink /etc/systemd/system/default.target;
-ln -sf /lib/systemd/system/graphical.target /etc/systemd/system/default.target;
-yum -y install file-roller
-
-# NetworkMiner has a compatibility issue with Mono 6 right now
-if ! grep -q "NetworkMiner has a compatibility issue with Mono 6 right now" /etc/yum/pluginconf.d/versionlock.list; then
-
-cat << EOF >> /etc/yum/pluginconf.d/versionlock.list
-
-# NetworkMiner has a compatibility issue with Mono 6 right now
-0:mono-complete-4.2.1.102-0.xamarin.1.*
-0:mono-core-4.2.1.102-0.xamarin.1.*
-0:mono-data-4.2.1.102-0.xamarin.1.*
-0:mono-data-oracle-4.2.1.102-0.xamarin.1.*
-0:mono-data-sqlite-4.2.1.102-0.xamarin.1.*
-0:mono-devel-4.2.1.102-0.xamarin.1.*
-0:mono-extras-4.2.1.102-0.xamarin.1.*
-0:mono-locale-extras-4.2.1.102-0.xamarin.1.*
-0:mono-mvc-4.2.1.102-0.xamarin.1.*
-0:mono-nunit-4.2.1.102-0.xamarin.1.*
-0:mono-reactive-4.2.1.102-0.xamarin.1.*
-0:mono-wcf-4.2.1.102-0.xamarin.1.*
-0:mono-web-4.2.1.102-0.xamarin.1.*
-0:mono-winforms-4.2.1.102-0.xamarin.1.*
-0:mono-winfxcore-4.2.1.102-0.xamarin.1.*
-EOF
-
-fi
-
-# Install Mono - prereq for NetworkMiner
-rpmkeys --import "http://pool.sks-keyservers.net/pks/lookup?op=get&search=0x3fa7e0328081bff6a14da29aa6a19b38d3d831ef";
-curl https://download.mono-project.com/repo/centos7-stable.repo | tee /etc/yum.repos.d/mono-centos7-stable.repo;
-yum -y install mono-core mono-basic mono-winforms expect
-
-# Install NetworkMiner
-yum -y install libcanberra-gtk2;
-wget https://www.netresec.com/?download=NetworkMiner_2-4 -O /tmp/nm.zip;
-mkdir -p /opt/networkminer/
-unzip /tmp/nm.zip -d /opt/networkminer/;
-rm /tmp/nm.zip;
-mv /opt/networkminer/NetworkMiner_*/* /opt/networkminer/
-chmod +x /opt/networkminer/NetworkMiner.exe;
-chmod -R go+w /opt/networkminer/AssembledFiles/;
-chmod -R go+w /opt/networkminer/Captures/;
-# Create networkminer shim
-cat << EOF >> /bin/networkminer
-#!/bin/bash
-/bin/mono /opt/networkminer/NetworkMiner.exe --noupdatecheck "\$@"
-EOF
-chmod +x /bin/networkminer
-# Convert networkminer ico file to png format
-yum -y install ImageMagick
-convert /opt/networkminer/networkminericon.ico /opt/networkminer/networkminericon.png
-# Create menu entry
-cat << EOF >> /usr/share/applications/networkminer.desktop
-[Desktop Entry]
-Name=NetworkMiner
-Comment=NetworkMiner
-Encoding=UTF-8
-Exec=/bin/networkminer %f
-Icon=/opt/networkminer/networkminericon-4.png
-StartupNotify=true
-Terminal=false
-X-MultipleArgs=false
-Type=Application
-MimeType=application/x-pcap;
-Categories=Network;
-EOF
-
-# Set default monospace font to Liberation
-cat << EOF >> /etc/fonts/local.conf
- <match target="pattern">
-  <test name="family" qual="any">
-   <string>monospace</string>
-  </test>
-  <edit binding="strong" mode="prepend" name="family">
-   <string>Liberation Mono</string>
-  </edit>
- </match>
-EOF
-
-# Install Wireshark for Gnome
-yum -y install wireshark-gnome; 
-
-# Install dnsiff
-yum -y install dsniff;
-
-# Install hping3
-yum -y install hping3;
-
-# Install netsed
-yum -y install netsed;
-
-# Install ngrep
-yum -y install ngrep;
-
-# Install scapy
-yum -y install python36-scapy;
-
-# Install ssldump
-yum -y install ssldump;
-
-# Install tcpdump
-yum -y install tcpdump;
-
-# Install tcpflow
-yum -y install tcpflow;
-
-# Install tcpxtract
-yum -y install tcpxtract;
-
-# Install whois
-yum -y install whois;
-
-# Install foremost
-yum -y install https://forensics.cert.org/centos/cert/7/x86_64//foremost-1.5.7-13.1.el7.x86_64.rpm;
-
-# Install chromium
-yum -y install chromium;
-
-# Install tcpstat
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-tcpstat-1.5.0/securityonion-tcpstat-1.5.0.rpm;
-
-# Install tcptrace
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-tcptrace-6.6.7/securityonion-tcptrace-6.6.7.rpm;
-
-# Install sslsplit
-yum -y install libevent;
-yum -y install sslsplit;
-
-# Install Bit-Twist
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-bittwist-2.0.0/securityonion-bittwist-2.0.0.rpm;
-
-# Install chaosreader
-yum -y install perl-IO-Compress perl-Net-DNS;
-yum -y install https://github.com/Security-Onion-Solutions/securityonion-docker-rpm/releases/download/securityonion-chaosreader-0.95.10/securityonion-chaosreader-0.95.10.rpm;
-chmod +x /bin/chaosreader;
-
-cp ../files/analyst/README /;
-
-echo
-echo "Analyst workstation has been installed!"
-echo "Press ENTER to reboot or Ctrl-C to cancel."
-read pause
-
-reboot;

setup/so-functions

@@ -1104,7 +1104,6 @@ manager_pillar() {
 		"logstash_settings:"\
 		"  ls_pipeline_batch_size: 125"\
 		"  ls_input_threads: 1"\
-		"  ls_batch_count: 125"\
 		"  lsheap: $LS_HEAP_SIZE"\
 		"  ls_pipeline_workers: $num_cpu_cores"\
 		""\
@@ -1333,8 +1332,8 @@ elasticsearch_pillar() {
 		"logstash_settings:"\
 		"  ls_pipeline_batch_size: $LSPIPELINEBATCH"\
 		"  ls_input_threads: $LSINPUTTHREADS"\
-		"  ls_batch_count: $LSINPUTBATCHCOUNT"\
 		"  lsheap: $NODE_LS_HEAP_SIZE"\
+		"  ls_pipeline_workers: $num_cpu_cores"\
 		"" >> "$pillar_file"
 
 }

setup/so-setup

@@ -272,7 +272,7 @@ if [[ $is_manager && $is_node ]]; then
 	LSPIPELINEWORKERS=1
 	LSPIPELINEBATCH=125
 	LSINPUTTHREADS=1
-	LSINPUTBATCHCOUNT=125
+	LSPIPELINEBATCH=125
 	NIDS=Suricata
 	ZEEKVERSION=ZEEK
 fi
@@ -387,7 +387,6 @@ if [[ $is_node && ! $is_eval ]]; then
 		whiptail_node_ls_pipeline_worker
 		whiptail_node_ls_pipline_batchsize
 		whiptail_node_ls_input_threads
-		whiptail_node_ls_input_batch_count
 		whiptail_cur_close_days
 		whiptail_log_size_limit
 	else
@@ -396,7 +395,7 @@ if [[ $is_node && ! $is_eval ]]; then
 		LSPIPELINEWORKERS=$num_cpu_cores
 		LSPIPELINEBATCH=125
 		LSINPUTTHREADS=1
-		LSINPUTBATCHCOUNT=125
+		LSPIPELINEBATCH=125
 	fi
 fi
 
@@ -519,14 +518,9 @@ fi
 		
 		set_progress_str 12 'Generating manager pillar'
 		manager_pillar >> $setup_log 2>&1
-	fi
-
-	if [[ $is_sensor || $is_import ]]; then
-		set_progress_str 13 'Generating zeeklogs pillar'
 		zeek_logs_enabled >> $setup_log 2>&1
 	fi
-
+	
-
 	set_progress_str 16 'Running first Salt checkin'
 	salt_firstcheckin >> $setup_log 2>&1
 
@@ -623,7 +617,6 @@ fi
 		salt-call state.apply -l info suricata >> $setup_log 2>&1
 
 		set_progress_str 67 "$(print_salt_state_apply 'zeek')"
-		zeek_logs_enabled >> $setup_log 2>&1
 		salt-call state.apply -l info zeek >> $setup_log 2>&1
 	fi
 

setup/so-whiptail

@@ -896,19 +896,6 @@ whiptail_node_ls_input_threads() {
 
 }
 
-whiptail_node_ls_input_batch_count() {
-
-	[ -n "$TESTING" ] && return
-
-	LSINPUTBATCHCOUNT=$(whiptail --title "Security Onion Setup" --inputbox \
-	"\nEnter LogStash Input Batch Count: \n \n(Default value is pre-populated)" 10 75 125 3>&1 1>&2 2>&3)
-
-	local exitstatus=$?
-	whiptail_check_exitstatus $exitstatus
-
-}
-
-
 #TODO: helper function to display error message or exit if batch mode
 # exit_if_batch <"Error string"> <Error code (int)>
 

so-analyst-install

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+cd "$(dirname "$0")/salt/common/tools/sbin" || exit 255
+
+./so-analyst-install