evalfix

pillar/logstash/eval.sls

@@ -0,0 +1,4 @@
+logstash:
+  pipelines:
+    eval:
+      config: "/usr/share/logstash/pipelines/eval/*.conf"

pillar/logstash/search.sls

@@ -1,6 +1,4 @@
 logstash:
   pipelines:
-    master:
-      config: "/usr/share/logstash/pipelines/master/*.conf"
     search:
       config: "/usr/share/logstash/pipelines/search/*.conf"

pillar/top.sls

@@ -4,7 +4,8 @@ base:
 
   'G@role:so-mastersearch or G@role:so-heavynode':
     - match: compound
-    - logstash.mastersearch
+    - logstash.master
+    - logstash.search
 
   'G@role:so-sensor':
     - static
@@ -29,6 +30,7 @@ base:
     - data.*
     - brologs
     - auth
+    - logstash.eval
     - minions.{{ grains.id }}
 
   'G@role:so-node':

salt/auth/init.sls

@@ -1,3 +1,6 @@
+{% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %}
+{% set MASTER = salt['grains.get']('master') %}
+
 so-auth-api-dir:
   file.directory:
     - name: /opt/so/conf/auth/api
@@ -5,19 +8,9 @@ so-auth-api-dir:
     - group: 939
     - makedirs: True
 
-so-auth-api-image:
-    cmd.run:
-        - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-auth-api:HH1.1.4
-
-so-auth-ui-image:
-    cmd.run:
-        - name: docker pull --disable-content-trust=false docker.io/soshybridhunter/so-auth-ui:HH1.1.4
-
 so-auth-api:
     docker_container.running:
-        - require:
-            - so-auth-api-image
-        - image: docker.io/soshybridhunter/so-auth-api:HH1.1.4
+        - image: {{ MASTER }}:5000/soshybridhunter/so-auth-api:{{ VERSION }}
         - hostname: so-auth-api
         - name: so-auth-api
         - environment:
@@ -29,9 +22,7 @@ so-auth-api:
 
 so-auth-ui:
     docker_container.running:
-        - require:
-            - so-auth-ui-image
-        - image: docker.io/soshybridhunter/so-auth-ui:HH1.1.4
+        - image: {{ MASTER }}:5000/soshybridhunter/so-auth-ui:{{ VERSION }}
         - hostname: so-auth-ui
         - name: so-auth-ui
         - port_bindings:

salt/filebeat/etc/filebeat.yml

@@ -1,5 +1,5 @@
 {%- if grains.role == 'so-heavynode' %}
-{%- set MASTER = grains.host %}
+{%- set MASTER = salt['pillar.get']('sensor:mainip' '') %}
 {%- else %}
 {%- set MASTER = grains['master'] %}
 {%- endif %}

salt/filebeat/init.sls

@@ -13,14 +13,13 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 {% set VERSION = salt['pillar.get']('static:soversion', 'HH1.1.4') %}
 {% set MASTER = salt['grains.get']('master') %}
-{%- set MASTERIP = salt['pillar.get']('static:masterip', '') %}
+{% set MASTERIP = salt['pillar.get']('static:masterip', '') %}
 {% set FEATURES = salt['pillar.get']('elastic:features', False) %}
 {% if FEATURES %}
   {% set FEATURES = "-features" %}
 {% else %}
   {% set FEATURES = '' %}
 {% endif %}
-# Filebeat Setup
 filebeatetcdir:
   file.directory:
     - name: /opt/so/conf/filebeat/etc

salt/logstash/conf/pipelines/eval/0010_input_hhbeats.conf

@@ -1,40 +0,0 @@
-input {
-  beats {
-    port => "5644"
-    ssl => true
-    ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"]
-    ssl_certificate => "/usr/share/logstash/filebeat.crt"
-    ssl_key => "/usr/share/logstash/filebeat.key"
-    tags => [ "beat" ]
-  }
-}
-filter {
-  if [type] == "ids" or [type] =~ "bro" {
-    mutate {
-      rename => { "host" => "beat_host" }
-      remove_tag => ["beat"]
-      add_field => { "sensor_name" => "%{[beat][name]}" }
-      add_field => { "syslog-host_from" => "%{[beat][name]}" }
-      remove_field => [ "beat", "prospector", "input", "offset" ]
-    }
-  }
-  if [type] =~ "ossec" {
-    mutate {
-      rename => { "host" => "beat_host" }
-      remove_tag => ["beat"]
-      add_field => { "syslog-host_from" => "%{[beat][name]}" }
-      remove_field => [ "beat", "prospector", "input", "offset" ]
-    }
-   }
-    if [type] == "osquery" {
-    mutate {
-      rename => { "host" => "beat_host" }
-      remove_tag => ["beat"]
-      add_tag => ["osquery"]
-        	}
-   json {
-    source => "message"
-    target => "osquery"
-        }
-  }
-}

salt/logstash/conf/pipelines/eval/templates/0800_input_eval.conf

@@ -9,173 +9,173 @@ input {
 	}
 	file {
 		path => "/nsm/zeek/logs/current/conn*.log"
-		type => "zeek_conn"
-	    	tags => ["zeek"]
+		type => "bro_conn"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/dce_rpc*.log"
-		type => "zeek_dce_rpc"
-	    	tags => ["zeek"]
+		type => "bro_dce_rpc"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/dhcp*.log"
-		type => "zeek_dhcp"
-	    	tags => ["zeek"]
+		type => "bro_dhcp"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/dnp3*.log"
-		type => "zeek_dnp3"
-	    	tags => ["zeek"]
+		type => "bro_dnp3"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/dns*.log"
-		type => "zeek_dns"
-	    	tags => ["zeek"]
+		type => "bro_dns"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/dpd*.log"
-		type => "zeek_dpd"
-	    	tags => ["zeek"]
+		type => "bro_dpd"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/files*.log"
-		type => "zeek_files"
-	    	tags => ["zeek"]
+		type => "bro_files"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/ftp*.log"
-		type => "zeek_ftp"
-	    	tags => ["zeek"]
+		type => "bro_ftp"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/http*.log"
-		type => "zeek_http"
-	    	tags => ["zeek"]
+		type => "bro_http"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/intel*.log"
-		type => "zeek_intel"
-	    	tags => ["zeek"]
+		type => "bro_intel"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/irc*.log"
-		type => "zeek_irc"
-	    	tags => ["zeek"]
+		type => "bro_irc"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/kerberos*.log"
-		type => "zeek_kerberos"
-	    	tags => ["zeek"]
+		type => "bro_kerberos"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/modbus*.log"
-		type => "zeek_modbus"
-	    	tags => ["zeek"]
+		type => "bro_modbus"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/mysql*.log"
-		type => "zeek_mysql"
-	    	tags => ["zeek"]
+		type => "bro_mysql"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/notice*.log"
-		type => "zeek_notice"
-	    	tags => ["zeek"]
+		type => "bro_notice"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/ntlm*.log"
-		type => "zeek_ntlm"
-	    	tags => ["zeek"]
+		type => "bro_ntlm"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/pe*.log"
-		type => "zeek_pe"
-	    	tags => ["zeek"]
+		type => "bro_pe"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/radius*.log"
-		type => "zeek_radius"
-	    	tags => ["zeek"]
+		type => "bro_radius"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/rdp*.log"
-		type => "zeek_rdp"
-	    	tags => ["zeek"]
+		type => "bro_rdp"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/rfb*.log"
-		type => "zeek_rfb"
-	    	tags => ["zeek"]
+		type => "bro_rfb"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/signatures*.log"
-		type => "zeek_signatures"
-	    	tags => ["zeek"]
+		type => "bro_signatures"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/sip*.log"
-		type => "zeek_sip"
-	    	tags => ["zeek"]
+		type => "bro_sip"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/smb_files*.log"
-		type => "zeek_smb_files"
-	    	tags => ["zeek"]
+		type => "bro_smb_files"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/smb_mapping*.log"
-		type => "zeek_smb_mapping"
-	    	tags => ["zeek"]
+		type => "bro_smb_mapping"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/smtp*.log"
-		type => "zeek_smtp"
-	    	tags => ["zeek"]
+		type => "bro_smtp"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/snmp*.log"
-		type => "zeek_snmp"
-	    	tags => ["zeek"]
+		type => "bro_snmp"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/socks*.log"
-		type => "zeek_socks"
-	    	tags => ["zeek"]
+		type => "bro_socks"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/software*.log"
-		type => "zeek_software"
-	    	tags => ["zeek"]
+		type => "bro_software"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/ssh*.log"
-		type => "zeek_ssh"
-	    	tags => ["zeek"]
+		type => "bro_ssh"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/ssl*.log"
-		type => "zeek_ssl"
-	    	tags => ["zeek"]
+		type => "bro_ssl"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/syslog*.log"
-		type => "zeek_syslog"
-	    	tags => ["zeek"]
+		type => "bro_syslog"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/tunnel*.log"
-		type => "zeek_tunnels"
-	    	tags => ["zeek"]
+		type => "bro_tunnels"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/weird*.log"
-		type => "zeek_weird"
-	    	tags => ["zeek"]
+		type => "bro_weird"
+	    	tags => ["bro"]
 	}
 	file {
 		path => "/nsm/zeek/logs/current/x509*.log"
-		type => "zeek_x509"
-	    	tags => ["zeek"]
+		type => "bro_x509"
+	    	tags => ["bro"]
 	}
   file {
     path => "/wazuh/alerts/alerts.json"

salt/logstash/files/dynamic/0008_input_eval.conf

@@ -0,0 +1,203 @@
+# Updated by: Mike Reeves
+# Last Update: 11/1/2018
+
+input {
+  file {
+	  path => "/suricata/eve.json"
+		type => "ids"
+                add_field => { "engine" => "suricata" }
+	}
+	file {
+		path => "/nsm/bro/logs/current/conn*.log"
+		type => "bro_conn"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/dce_rpc*.log"
+		type => "bro_dce_rpc"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/dhcp*.log"
+		type => "bro_dhcp"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/dnp3*.log"
+		type => "bro_dnp3"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/dns*.log"
+		type => "bro_dns"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/dpd*.log"
+		type => "bro_dpd"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/files*.log"
+		type => "bro_files"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/ftp*.log"
+		type => "bro_ftp"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/http*.log"
+		type => "bro_http"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/intel*.log"
+		type => "bro_intel"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/irc*.log"
+		type => "bro_irc"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/kerberos*.log"
+		type => "bro_kerberos"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/modbus*.log"
+		type => "bro_modbus"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/mysql*.log"
+		type => "bro_mysql"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/notice*.log"
+		type => "bro_notice"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/ntlm*.log"
+		type => "bro_ntlm"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/pe*.log"
+		type => "bro_pe"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/radius*.log"
+		type => "bro_radius"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/rdp*.log"
+		type => "bro_rdp"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/rfb*.log"
+		type => "bro_rfb"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/signatures*.log"
+		type => "bro_signatures"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/sip*.log"
+		type => "bro_sip"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/smb_files*.log"
+		type => "bro_smb_files"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/smb_mapping*.log"
+		type => "bro_smb_mapping"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/smtp*.log"
+		type => "bro_smtp"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/snmp*.log"
+		type => "bro_snmp"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/socks*.log"
+		type => "bro_socks"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/software*.log"
+		type => "bro_software"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/ssh*.log"
+		type => "bro_ssh"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/ssl*.log"
+		type => "bro_ssl"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/syslog*.log"
+		type => "bro_syslog"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/tunnel*.log"
+		type => "bro_tunnels"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/weird*.log"
+		type => "bro_weird"
+	    	tags => ["bro"]
+	}
+	file {
+		path => "/nsm/bro/logs/current/x509*.log"
+		type => "bro_x509"
+	    	tags => ["bro"]
+	}
+  file {
+    path => "/wazuh/alerts/alerts.json"
+    type => "ossec"
+  }
+  file {
+    path => "/wazuh/archives/archive.json"
+    type => "ossec_archive"
+  }
+  file {
+    path => "/osquery/logs/result.log"
+    type => "osquery"
+  }
+  file {
+    path => "/strelka/strelka.log"
+    type => "strelka"
+  }
+}
+filter {
+  if "import" in [tags] {
+	mutate {
+	  #add_tag => [ "conf_file_0007"]
+	}
+  }
+}

salt/ssl/init.sls

@@ -2,7 +2,7 @@
 {% set master_minion_id = master.split(".")[0] %}
 {%- set masterip = salt['pillar.get']('static:masterip', '') -%}
 
-{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval'  or grains['role'] == 'so-heavynode' %}
+{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' %}
     {% set trusttheca_text =  salt['mine.get'](grains.id, 'x509.get_pem_entries')[grains.id]['/etc/pki/ca.crt']|replace('\n', '') %}
     {% set ca_server = grains.id %}
 {% else %}
@@ -41,7 +41,7 @@ m2cryptopkgs:
         bits: 4096
         backup: True
 
-{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' or grains['role'] == 'so-heavynode' %}
+{% if grains['role'] == 'so-master' or grains['role'] == 'so-eval' or grains['role'] == 'so-helix' or grains['role'] == 'so-mastersearch' %}
 
 # Request a cert and drop it where it needs to go to be distributed
 /etc/pki/filebeat.crt:

setup/functions.sh

@@ -651,7 +651,7 @@ install_master() {
 ls_heapsize() {
 
   # Determine LS Heap Size
-  if [ $TOTAL_MEM -ge 32000 ] ; then
+  if [ $TOTAL_MEM -ge 32000 ] || [ $INSTALLTYPE == 'EVALMODE' ]; then
       LS_HEAP_SIZE="1000m"
   else
       # If minimal RAM, then set minimal heap

setup/so-setup.sh

@@ -843,7 +843,7 @@ if (whiptail_you_sure) ; then
       whiptail_log_size_limit
     else
       NODE_ES_HEAP_SIZE=$ES_HEAP_SIZE
-      NODE_LS_HEAP_SIZE=1000m
+      NODE_LS_HEAP_SIZE=$LS_HEAP_SIZE
       LSPIPELINEWORKERS=$CPUCORES
       LSPIPELINEBATCH=125
       LSINPUTTHREADS=1