Merge remote-tracking branch 'remotes/origin/dev' into feature/metasuri

pillar/logstash/search.sls

@@ -5,6 +5,7 @@ logstash:
         - so/0900_input_redis.conf.jinja
         - so/9000_output_zeek.conf.jinja
         - so/9002_output_import.conf.jinja
+        - so/9034_output_syslog.conf.jinja
         - so/9100_output_osquery.conf.jinja
         - so/9400_output_suricata.conf.jinja
         - so/9500_output_beats.conf.jinja

salt/common/init.sls

@@ -62,6 +62,7 @@ commonpkgs:
       - python3-dateutil
       - python3-m2crypto
       - python3-mysqldb
+      - git
 heldpackages:
   pkg.installed:
     - pkgs:
@@ -96,6 +97,7 @@ commonpkgs:
       - device-mapper-persistent-data
       - lvm2
       - openssl
+      - git
 
 heldpackages:
   pkg.installed:

salt/curator/init.sls

@@ -89,7 +89,7 @@ curdel:
 
 so-curatorcloseddeletecron:
  cron.present:
-   - name: /usr/sbin/so-curator-closed-delete
+   - name: /usr/sbin/so-curator-closed-delete > /opt/so/log/curator/cron-closed-delete.log 2>&1
    - user: root
    - minute: '*'
    - hour: '*'
@@ -99,7 +99,7 @@ so-curatorcloseddeletecron:
 
 so-curatorclosecron:
  cron.present:
-   - name: /usr/sbin/so-curator-close
+   - name: /usr/sbin/so-curator-close > /opt/so/log/curator/cron-close.log 2>&1
    - user: root
    - minute: '*'
    - hour: '*'
@@ -109,7 +109,7 @@ so-curatorclosecron:
 
 so-curatordeletecron:
  cron.present:
-   - name: /usr/sbin/so-curator-delete
+   - name: /usr/sbin/so-curator-delete > /opt/so/log/curator/cron-delete.log 2>&1
    - user: root
    - minute: '*'
    - hour: '*'

salt/elasticsearch/files/ingest/syslog

@@ -6,6 +6,10 @@
                 "field": "message",
                 "pattern" : "%{message}",
                 "on_failure": [ { "drop" : { } } ]
+        },
+        "remove": {
+                "field": [ "type", "agent" ],
+                "ignore_failure": true
         }
     },
     { "pipeline":       { "name": "common"                                             } }

salt/filebeat/etc/filebeat.yml

@@ -75,9 +75,9 @@ filebeat.modules:
 filebeat.inputs:
 #------------------------------ Log prospector --------------------------------
 {%- if grains['role'] == 'so-sensor' or grains['role'] == "so-eval" or grains['role'] == "so-helix" or grains['role'] == "so-heavynode" or grains['role'] == "so-standalone" %}
-  - type: syslog
+
+  - type: udp
     enabled: true
-    protocol.udp:
     host: "0.0.0.0:514"
     fields:
       module: syslog
@@ -87,7 +87,20 @@ filebeat.inputs:
     processors:
     - drop_fields:
         fields: ["source", "prospector", "input", "offset", "beat"]
+    fields_under_root: true
 
+  - type: tcp
+    enabled: true
+    host: "0.0.0.0:514"
+    fields:
+      module: syslog
+      dataset: syslog
+    pipeline: "syslog"
+    index: "so-syslog-%{+yyyy.MM.dd}"
+    processors:
+    - drop_fields:
+        fields: ["source", "prospector", "input", "offset", "beat"]
+    fields_under_root: true
 {%- if BROVER != 'SURICATA' %}
 {%- for LOGNAME in salt['pillar.get']('brologs:enabled', '')  %}
   - type: log

salt/idstools/init.sls

@@ -39,7 +39,7 @@ idstoolsetcsync:
 
 so-ruleupdatecron:
   cron.present:
-    - name: /usr/sbin/so-rule-update.sh > /opt/so/log/idstools/download.log
+    - name: /usr/sbin/so-rule-update > /opt/so/log/idstools/download.log 2>&1
     - user: root
     - minute: '1'
     - hour: '7'

salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja

@@ -3,24 +3,21 @@
 {%- else %}
 {%- set ES = salt['pillar.get']('node:mainip', '') -%}
 {%- endif %}
-# Author: Justin Henderson
-#         SANS Instructor and author of SANS SEC555: SIEM and Tactical Analytics
-# Updated by: Doug Burks
-# Last Update: 5/15/2017
 
 filter {
-  if "syslog" in [tags] and "test_data" not in [tags] {
+  if [module] =~ "syslog" {
     mutate {
-	  ##add_tag => [ "conf_file_9034"]
+          ##add_tag => [ "conf_file_9000"]
         }
   }
 }
 output {
-  if "syslog" in [tags] and "test_data" not in [tags] {
+  if [module] =~ "syslog" {
     elasticsearch {
+      pipeline => "%{module}"
       hosts => "{{ ES }}"
       index => "so-syslog-%{+YYYY.MM.dd}"
-      template_name => "logstash"
+      template_name => "so-common"
       template => "/so-common-template.json"
       template_overwrite => true
     }

salt/nginx/etc/nginx.conf.so-eval

@@ -134,7 +134,7 @@ http {
           proxy_set_header      Connection "Upgrade";
         }
 
-        location ~ ^/auth/.*?(whoami|login|logout) {
+        location ~ ^/auth/.*?(whoami|login|logout|settings) {
           rewrite               /auth/(.*) /$1 break;
           proxy_pass            http://{{ masterip }}:4433;
           proxy_read_timeout    90;

salt/nginx/etc/nginx.conf.so-master

@@ -134,7 +134,7 @@ http {
           proxy_set_header      Connection "Upgrade";
         }
 
-        location ~ ^/auth/.*?(whoami|login|logout) {
+        location ~ ^/auth/.*?(whoami|login|logout|settings) {
           rewrite               /auth/(.*) /$1 break;
           proxy_pass            http://{{ masterip }}:4433;
           proxy_read_timeout    90;

salt/nginx/etc/nginx.conf.so-mastersearch

@@ -134,7 +134,7 @@ http {
           proxy_set_header      Connection "Upgrade";
         }
 
-        location ~ ^/auth/.*?(whoami|login|logout) {
+        location ~ ^/auth/.*?(whoami|login|logout|settings) {
           rewrite               /auth/(.*) /$1 break;
           proxy_pass            http://{{ masterip }}:4433;
           proxy_read_timeout    90;

salt/playbook/init.sls

@@ -86,15 +86,22 @@ so-playbook:
 
 {% endif %}
 
+playbooklogdir:
+  file.directory:
+    - name: /opt/so/log/playbook
+    - user: 939
+    - group: 939
+    - makedirs: True
+
 so-playbooksynccron:
   cron.present:
-    - name: /usr/sbin/so-playbook-sync
+    - name: /usr/sbin/so-playbook-sync > /opt/so/log/playbook/sync.log 2>&1
     - user: root
     - minute: '*/5'
 
 so-playbookruleupdatecron:
   cron.present:
-    - name: /usr/sbin/so-playbook-ruleupdate
+    - name: /usr/sbin/so-playbook-ruleupdate > /opt/so/log/playbook/update.log 2>&1
     - user: root
     - minute: '1'
     - hour: '6'

salt/soc/files/kratos/kratos.yaml

@@ -42,7 +42,7 @@ urls:
   login_ui: https://{{ WEBACCESS }}/login/
   registration_ui: https://{{ WEBACCESS }}/login/
   error_ui: https://{{ WEBACCESS }}/login/
-  settings_ui: https://{{ WEBACCESS }}/
+  settings_ui: https://{{ WEBACCESS }}/?r=/settings
   verify_ui: https://{{ WEBACCESS }}/
   mfa_ui: https://{{ WEBACCESS }}/
 

salt/wazuh/init.sls

@@ -80,11 +80,6 @@ wazuhmgrwhitelist:
     - mode: 755
     - template: jinja
 
-wazuhagentservice:
-  service.running:
-    - name: wazuh-agent
-    - enable: True
-
 so-wazuh:
   docker_container.running:
     - image: {{ MASTER }}:5000/soshybridhunter/so-wazuh:{{ VERSION }}
@@ -110,3 +105,8 @@ whitelistmanager:
   cmd.run:
     - name: /usr/sbin/wazuh-manager-whitelist
     - cwd: /
+
+wazuhagentservice:
+  service.running:
+    - name: wazuh-agent
+    - enable: True

salt/zeek/cron/packetloss.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-/usr/bin/docker exec so-zeek /opt/zeek/bin/zeekctl netstats | awk '{print $(NF-2),$(NF-1),$NF}' | awk -F '[ =]' '{RCVD += $2;DRP += $4;TTL += $6} END { print "rcvd: " RCVD, "dropped: " DRP, "total: " TTL}' >> /nsm/zeek/logs/packetloss.log
+/usr/bin/docker exec so-zeek /opt/zeek/bin/zeekctl netstats | awk '{print $(NF-2),$(NF-1),$NF}' | awk -F '[ =]' '{RCVD += $2;DRP += $4;TTL += $6} END { print "rcvd: " RCVD, "dropped: " DRP, "total: " TTL}' >> /nsm/zeek/logs/packetloss.log 2>&1

setup/so-functions

@@ -1217,7 +1217,7 @@ salt_checkin() {
 
 # Run a salt command to generate the minion key
 salt_firstcheckin() {
-	salt-call state.show_top >> /dev/null # send output to /dev/null because we don't actually care about the ouput
+	salt-call state.show_top >> /dev/null 2>&1 # send output to /dev/null because we don't actually care about the ouput
 }
 
 set_base_heapsizes() {

setup/so-whiptail

@@ -24,7 +24,7 @@ whiptail_basic_bro() {
 	[ -n "$TESTING" ] && return
 
 	BASICBRO=$(whiptail --title "Security Onion Setup" --inputbox \
-	"Enter the number of bro processes:" 10 75 "$lb_procs" 3>&1 1>&2 2>&3)
+	"Enter the number of zeek processes:" 10 75 "$lb_procs" 3>&1 1>&2 2>&3)
 
 	local exitstatus=$?
 	whiptail_check_exitstatus $exitstatus
@@ -51,7 +51,7 @@ whiptail_bro_pins() {
 		cpu_core_list_whiptail+=("$item" "OFF")
 	done
 
-	BROPINS=$(whiptail --noitem --title "Pin Bro CPUS" --checklist "Please select $lb_procs cores to pin Bro to:" 20 75 12 "${cpu_core_list_whiptail[@]}" 3>&1 1>&2 2>&3 )
+	BROPINS=$(whiptail --noitem --title "Pin Zeek CPUS" --checklist "Please select $lb_procs cores to pin Zeek to:" 20 75 12 "${cpu_core_list_whiptail[@]}" 3>&1 1>&2 2>&3 )
 	local exitstatus=$?
 	whiptail_check_exitstatus $exitstatus