From 8ee2142de488348e7ceb39b891365c8e61f1e5ae Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Jun 2020 11:21:47 -0400 Subject: [PATCH 01/14] pillarize idstools - https://github.com/Security-Onion-Solutions/securityonion/issues/583 --- salt/idstools/etc/disable.conf | 6 ++++++ salt/idstools/etc/enable.conf | 6 ++++++ salt/idstools/init.sls | 2 ++ 3 files changed, 14 insertions(+) diff --git a/salt/idstools/etc/disable.conf b/salt/idstools/etc/disable.conf index e8928c495..9eb69575e 100644 --- a/salt/idstools/etc/disable.conf +++ b/salt/idstools/etc/disable.conf @@ -1,3 +1,5 @@ +{% set disabled_sids = salt['pillar.get']('idstools:sids:disabled', {}) -%} + # idstools - disable.conf # Example of disabling a rule by signature ID (gid is optional). @@ -8,3 +10,7 @@ # - All regular expression matches are case insensitive. # re:hearbleed # re:MS(0[7-9]|10)-\d+ + +{%- for sid in disabled_sids %} +{{ sid }} +{% endfor -%} \ No newline at end of file diff --git a/salt/idstools/etc/enable.conf b/salt/idstools/etc/enable.conf index b7738677c..3f965541f 100644 --- a/salt/idstools/etc/enable.conf +++ b/salt/idstools/etc/enable.conf @@ -1,3 +1,5 @@ +{% set enabled_sids = salt['pillar.get']('idstools:sids:enabled', {}) -%} + # idstools-rulecat - enable.conf # Example of enabling a rule by signature ID (gid is optional). @@ -8,3 +10,7 @@ # - All regular expression matches are case insensitive. # re:hearbleed # re:MS(0[7-9]|10)-\d+ + +{%- for sid in enabled_sids %} +{{ sid }} +{% endfor -%} \ No newline at end of file diff --git a/salt/idstools/init.sls b/salt/idstools/init.sls index d145a4e15..078cb5b03 100644 --- a/salt/idstools/init.sls +++ b/salt/idstools/init.sls @@ -66,3 +66,5 @@ so-idstools: - binds: - /opt/so/conf/idstools/etc:/opt/so/idstools/etc:ro - /opt/so/rules/nids:/opt/so/rules/nids:rw + - watch: + - file: idstoolsetcsync From fe8df2206309f5ca1a2f7e357978e74cee08a105 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Jun 2020 16:03:14 -0400 Subject: [PATCH 02/14] cleanup whitespace in idstools enable/disable.conf --- salt/idstools/etc/disable.conf | 1 - salt/idstools/etc/enable.conf | 1 - 2 files changed, 2 deletions(-) diff --git a/salt/idstools/etc/disable.conf b/salt/idstools/etc/disable.conf index 9eb69575e..33f7e0ae6 100644 --- a/salt/idstools/etc/disable.conf +++ b/salt/idstools/etc/disable.conf @@ -1,5 +1,4 @@ {% set disabled_sids = salt['pillar.get']('idstools:sids:disabled', {}) -%} - # idstools - disable.conf # Example of disabling a rule by signature ID (gid is optional). diff --git a/salt/idstools/etc/enable.conf b/salt/idstools/etc/enable.conf index 3f965541f..36f95fbc4 100644 --- a/salt/idstools/etc/enable.conf +++ b/salt/idstools/etc/enable.conf @@ -1,5 +1,4 @@ {% set enabled_sids = salt['pillar.get']('idstools:sids:enabled', {}) -%} - # idstools-rulecat - enable.conf # Example of enabling a rule by signature ID (gid is optional). From 67f2eedad1616dfdf6298aa11e5f7df447d39c37 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Jun 2020 16:11:30 -0400 Subject: [PATCH 03/14] cleanup whitespace in idstools enable/disable.conf --- salt/idstools/etc/disable.conf | 2 +- salt/idstools/etc/enable.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/idstools/etc/disable.conf b/salt/idstools/etc/disable.conf index 33f7e0ae6..ff7197360 100644 --- a/salt/idstools/etc/disable.conf +++ b/salt/idstools/etc/disable.conf @@ -12,4 +12,4 @@ {%- for sid in disabled_sids %} {{ sid }} -{% endfor -%} \ No newline at end of file +{%- endfor %} \ No newline at end of file diff --git a/salt/idstools/etc/enable.conf b/salt/idstools/etc/enable.conf index 36f95fbc4..c16b25986 100644 --- a/salt/idstools/etc/enable.conf +++ b/salt/idstools/etc/enable.conf @@ -12,4 +12,4 @@ {%- for sid in enabled_sids %} {{ sid }} -{% endfor -%} \ No newline at end of file +{%- endfor %} \ No newline at end of file From efaf41107c87021478ba8a903991edb6f005ec62 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Jun 2020 16:14:36 -0400 Subject: [PATCH 04/14] update description in localrules/local.rules for idstools --- salt/idstools/localrules/local.rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/idstools/localrules/local.rules b/salt/idstools/localrules/local.rules index e01ea913c..95be9156f 100644 --- a/salt/idstools/localrules/local.rules +++ b/salt/idstools/localrules/local.rules @@ -1 +1 @@ -# Put your own custom Snort/Suricata rules in here. +# Put your own custom Snort/Suricata rules in /opt/so/saltstack/local/salt/idstools/localrules/. \ No newline at end of file From 0f9d8024f853c5a22d93c39bf80b2d578622914c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Jun 2020 16:23:04 -0400 Subject: [PATCH 05/14] add sensoroni to so-status output for sensors - https://github.com/Security-Onion-Solutions/securityonion/issues/878 --- salt/common/maps/eval.map.jinja | 3 ++- salt/common/maps/heavynode.map.jinja | 3 ++- salt/common/maps/sensor.map.jinja | 3 ++- salt/common/maps/standalone.map.jinja | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/salt/common/maps/eval.map.jinja b/salt/common/maps/eval.map.jinja index f868474c5..075344e82 100644 --- a/salt/common/maps/eval.map.jinja +++ b/salt/common/maps/eval.map.jinja @@ -14,6 +14,7 @@ 'so-zeek', 'so-curator', 'so-elastalert', - 'so-soctopus' + 'so-soctopus', + 'so-sensoroni' ] } %} \ No newline at end of file diff --git a/salt/common/maps/heavynode.map.jinja b/salt/common/maps/heavynode.map.jinja index 2b8257a6a..cbd0fc3b0 100644 --- a/salt/common/maps/heavynode.map.jinja +++ b/salt/common/maps/heavynode.map.jinja @@ -9,6 +9,7 @@ 'so-steno', 'so-suricata', 'so-wazuh', - 'so-filebeat + 'so-filebeat', + 'so-sensoroni' ] } %} \ No newline at end of file diff --git a/salt/common/maps/sensor.map.jinja b/salt/common/maps/sensor.map.jinja index f172dd55d..3f5ebe8eb 100644 --- a/salt/common/maps/sensor.map.jinja +++ b/salt/common/maps/sensor.map.jinja @@ -3,6 +3,7 @@ 'so-telegraf', 'so-steno', 'so-suricata', - 'so-filebeat' + 'so-filebeat', + 'so-sensoroni' ] } %} \ No newline at end of file diff --git a/salt/common/maps/standalone.map.jinja b/salt/common/maps/standalone.map.jinja index 3585d0df6..d66cad1f9 100644 --- a/salt/common/maps/standalone.map.jinja +++ b/salt/common/maps/standalone.map.jinja @@ -16,6 +16,7 @@ 'so-suricata', 'so-steno', 'so-dockerregistry', - 'so-soctopus' + 'so-soctopus', + 'so-sensoroni' ] } %} \ No newline at end of file From 8f5da66335285a9536afcc0df23af04a40ae4843 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 30 Jun 2020 03:02:38 +0000 Subject: [PATCH 06/14] Add null safe operator for query name --- salt/elasticsearch/files/ingest/zeek.dns | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/files/ingest/zeek.dns b/salt/elasticsearch/files/ingest/zeek.dns index 050c6d414..09ce7fd9f 100644 --- a/salt/elasticsearch/files/ingest/zeek.dns +++ b/salt/elasticsearch/files/ingest/zeek.dns @@ -23,7 +23,7 @@ { "rename": { "field": "message2.TTLs", "target_field": "dns.ttls", "ignore_missing": true } }, { "rename": { "field": "message2.rejected", "target_field": "dns.query.rejected", "ignore_missing": true } }, { "script": { "lang": "painless", "source": "ctx.dns.query.length = ctx.dns.query.name.length()", "ignore_failure": true } }, - { "pipeline": { "if": "ctx.dns.query.name.contains('.')", "name": "zeek.dns.tld"} }, + { "pipeline": { "if": "ctx.dns.query?.name != null && ctx.dns.query.name.contains('.')", "name": "zeek.dns.tld" } }, { "pipeline": { "name": "zeek.common" } } ] } From bf8798f1d13ee3c5bbf455122c19b0dcfaa6d7c1 Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 30 Jun 2020 03:04:01 +0000 Subject: [PATCH 07/14] Fix krb client/server cert subject parsing --- salt/elasticsearch/files/ingest/zeek.kerberos | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/zeek.kerberos b/salt/elasticsearch/files/ingest/zeek.kerberos index 33381cd2d..917b38b54 100644 --- a/salt/elasticsearch/files/ingest/zeek.kerberos +++ b/salt/elasticsearch/files/ingest/zeek.kerberos @@ -13,9 +13,9 @@ { "rename": { "field": "message2.cipher", "target_field": "kerberos.ticket.cipher", "ignore_missing": true } }, { "rename": { "field": "message2.forwardable", "target_field": "kerberos.ticket.forwardable", "ignore_missing": true } }, { "rename": { "field": "message2.renewable", "target_field": "kerberos.ticket.renewable", "ignore_missing": true } }, - { "rename": { "field": "message2.client_cert_subject", "target_field": "kerberos.client.certificate.subject", "ignore_missing": true } }, + { "rename": { "field": "message2.client_cert_subject", "target_field": "kerberos.client_certificate.subject", "ignore_missing": true } }, { "rename": { "field": "message2.client_cert_fuid", "target_field": "log.id.client_certificate_fuid", "ignore_missing": true } }, - { "rename": { "field": "message2.server_cert_subject", "target_field": "kerberos.server.certificate.subject", "ignore_missing": true } }, + { "rename": { "field": "message2.server_cert_subject", "target_field": "kerberos.server_certificate.subject", "ignore_missing": true } }, { "rename": { "field": "message2.server_cert_fuid", "target_field": "log.id.server_certificate_fuid", "ignore_missing": true } }, { "pipeline": { "name": "zeek.common" } } ] From 84e2965fef6c1d03c7edba3a116a136113f8f8dc Mon Sep 17 00:00:00 2001 From: Wes Lambert Date: Tue, 30 Jun 2020 03:06:01 +0000 Subject: [PATCH 08/14] Addl krb fix --- salt/elasticsearch/files/ingest/zeek.kerberos | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/files/ingest/zeek.kerberos b/salt/elasticsearch/files/ingest/zeek.kerberos index 917b38b54..0b09ebf61 100644 --- a/salt/elasticsearch/files/ingest/zeek.kerberos +++ b/salt/elasticsearch/files/ingest/zeek.kerberos @@ -13,9 +13,9 @@ { "rename": { "field": "message2.cipher", "target_field": "kerberos.ticket.cipher", "ignore_missing": true } }, { "rename": { "field": "message2.forwardable", "target_field": "kerberos.ticket.forwardable", "ignore_missing": true } }, { "rename": { "field": "message2.renewable", "target_field": "kerberos.ticket.renewable", "ignore_missing": true } }, - { "rename": { "field": "message2.client_cert_subject", "target_field": "kerberos.client_certificate.subject", "ignore_missing": true } }, + { "rename": { "field": "message2.client_cert_subject", "target_field": "kerberos.client_certificate_subject", "ignore_missing": true } }, { "rename": { "field": "message2.client_cert_fuid", "target_field": "log.id.client_certificate_fuid", "ignore_missing": true } }, - { "rename": { "field": "message2.server_cert_subject", "target_field": "kerberos.server_certificate.subject", "ignore_missing": true } }, + { "rename": { "field": "message2.server_cert_subject", "target_field": "kerberos.server_certificate_subject", "ignore_missing": true } }, { "rename": { "field": "message2.server_cert_fuid", "target_field": "log.id.server_certificate_fuid", "ignore_missing": true } }, { "pipeline": { "name": "zeek.common" } } ] From 1c0443458c3515418b5995d727ff47f4aeda9434 Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Tue, 30 Jun 2020 08:13:00 -0400 Subject: [PATCH 09/14] so-allow spacing fix --- salt/common/tools/sbin/so-allow | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-allow b/salt/common/tools/sbin/so-allow index e86210874..f069a3e4a 100755 --- a/salt/common/tools/sbin/so-allow +++ b/salt/common/tools/sbin/so-allow @@ -88,6 +88,7 @@ if [ "$SKIP" -eq 0 ]; then echo "[w] - Wazuh agent - port 1514/tcp/udp" echo "[p] - Wazuh API - port 55000/tcp" echo "[r] - Wazuh registration service - 1515/tcp" + echo "" echo "Please enter your selection (a - analyst, b - beats, o - osquery, w - wazuh):" read -r ROLE echo "Enter a single ip address or range to allow (example: 10.10.10.10 or 10.10.0.0/16):" From 9f398751922b7b46ef553678caa49c9171540c64 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Mon, 29 Jun 2020 15:53:21 -0400 Subject: [PATCH 10/14] [fix] Apply regex filter to hostname input --- setup/so-whiptail | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 2aa7bf10c..008d24e1f 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -951,16 +951,23 @@ whiptail_set_hostname() { HOSTNAME=$(cat /etc/hostname) + if [[ "$HOSTNAME" == *'localhost'* ]]; then HOSTNAME=securityonion; fi + HOSTNAME=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter the Hostname you would like to set." 10 75 "$HOSTNAME" 3>&1 1>&2 2>&3) + "Enter the hostname (not FQDN) you would like to set." 10 75 "$HOSTNAME" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus - while [[ "$HOSTNAME" == *'localhost'* ]] ; do - whiptail --title "Security Onion Setup" --msgbox "Please choose a hostname that doesn't contain localhost." 8 75 + while [[ $HOSTNAME == *'localhost'* || ! ( $HOSTNAME =~ ^[a-zA-Z0-9\-]*$ ) ]] ; do + local error_message + error_message=$(echo "Please choose a valid hostname. It cannot contain localhost; and must contain only \ + the ASCII letters 'a' through 'z' (case-insensitive), the digits '0' through '9', \ + and hyphen ('-')" | tr -d '\t') + whiptail --title "Security Onion Setup" \ + --msgbox "$error_message" 10 75 HOSTNAME=$(whiptail --title "Security Onion Setup" --inputbox \ - "Enter the Hostname you would like to set." 10 75 "$HOSTNAME" 3>&1 1>&2 2>&3) + "Enter the hostname (not FQDN) you would like to set." 10 75 "$HOSTNAME" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus done From 8d624e6adecc91b22cf93a67dd8137dfcf3ea9e8 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Tue, 30 Jun 2020 10:53:10 -0400 Subject: [PATCH 11/14] [fix] Move navigatordefaultlayer file.managed state to nginx sls --- salt/nginx/init.sls | 10 ++++++++++ salt/soctopus/init.sls | 10 ---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/salt/nginx/init.sls b/salt/nginx/init.sls index 2ead5587a..73f14a7ed 100644 --- a/salt/nginx/init.sls +++ b/salt/nginx/init.sls @@ -49,6 +49,16 @@ navigatorconfig: - makedirs: True - template: jinja +navigatordefaultlayer: + file.managed: + - name: /opt/so/conf/navigator/nav_layer_playbook.json + - source: salt://nginx/files/nav_layer_playbook.json + - user: 939 + - group: 939 + - makedirs: True + - replace: False + - template: jinja + so-nginx: docker_container.running: - image: {{ MASTER }}:5000/soshybridhunter/so-nginx:{{ VERSION }} diff --git a/salt/soctopus/init.sls b/salt/soctopus/init.sls index eb9114c58..6630b32b5 100644 --- a/salt/soctopus/init.sls +++ b/salt/soctopus/init.sls @@ -49,16 +49,6 @@ playbookrulessync: - group: 939 - template: jinja -navigatordefaultlayer: - file.managed: - - name: /opt/so/conf/navigator/nav_layer_playbook.json - - source: salt://nginx/files/nav_layer_playbook.json - - user: 939 - - group: 939 - - makedirs: True - - replace: False - - template: jinja - so-soctopus: docker_container.running: - image: {{ MASTER }}:5000/soshybridhunter/so-soctopus:{{ VERSION }} From 930f15eea536a433734cb6b3a42ca066a9e244bb Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 30 Jun 2020 14:56:08 -0400 Subject: [PATCH 12/14] Introduce so-import-pcap tool - WIP --- salt/common/tools/sbin/so-import-pcap | 221 ++++++++++++++++++++++++++ salt/filebeat/etc/filebeat.yml | 44 ++++- salt/filebeat/init.sls | 4 +- 3 files changed, 263 insertions(+), 6 deletions(-) create mode 100644 salt/common/tools/sbin/so-import-pcap diff --git a/salt/common/tools/sbin/so-import-pcap b/salt/common/tools/sbin/so-import-pcap new file mode 100644 index 000000000..3ca1f0277 --- /dev/null +++ b/salt/common/tools/sbin/so-import-pcap @@ -0,0 +1,221 @@ +#!/bin/bash +# +# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +{% set MASTER = salt['grains.get']('master') %} +{% set VERSION = salt['pillar.get']('static:soversion') %} +{%- set MASTERIP = salt['pillar.get']('static:masterip') -%} + +function usage { + cat << EOF +Usage: +Please supply at least one pcap file. + +For example, to import a single pcap named import.pcap: +so-import-pcap import.pcap + +To import multiple pcaps: +so-import-pcap import1.pcap import2.pcap + +** IMPORTANT ** +Security Onion installations contain processes that automatically discard old data. Therefore, imports of old network traffic might immediately be erased, unless those processes are first disabled. + +EOF +} + +function pcapinfo() { + PCAP=$1 + ARGS=$2 + docker run --rm -v $PCAP:/input.pcap --entrypoint capinfos {{ MASTER }}:5000/soshybridhunter/so-pcaptools:{{ VERSION }} /input.pcap $ARGS +} + +function pcapfix() { + PCAP=$1 + PCAP_OUT=$2 + docker run --rm -v $PCAP:/input.pcap -v $PCAP_OUT:$PCAP_OUT --entrypoint pcapfix {{ MASTER }}:5000/soshybridhunter/so-pcaptools:{{ VERSION }} /input.pcap -o $PCAP_OUT > /dev/null 2>&1 +} + +function suricata() { + PCAP=$1 + HASH=$2 + + NSM_PATH=/nsm/import/${HASH}/suricata + mkdir -p $NSM_PATH + chown suricata:socore $NSM_PATH + LOG_PATH=/opt/so/log/suricata/import/${HASH} + mkdir -p $LOG_PATH + chown suricata:suricata $LOG_PATH + docker run --rm \ + -v /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro \ + -v /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro \ + -v /opt/so/conf/suricata/rules:/etc/suricata/rules:ro \ + -v ${LOG_PATH}:/var/log/suricata/:rw \ + -v ${NSM_PATH}/:/nsm/:rw \ + -v $PCAP:/input.pcap:ro \ + -v /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro \ + {{ MASTER }}:5000/soshybridhunter/so-suricata:{{ VERSION }} \ + --runmode single -k none -r /input.pcap > $LOG_PATH/console.log 2>&1 +} + +function zeek() { + PCAP=$1 + HASH=$2 + + NSM_PATH=/nsm/import/${HASH}/zeek + mkdir -p $NSM_PATH/logs + mkdir -p $NSM_PATH/spool + mkdir -p $NSM_PATH/extracted + chown -R zeek:socore $NSM_PATH + docker run --rm \ + -v $NSM_PATH:/nsm:rw \ + -v $PCAP:/input.pcap:ro \ + -v /opt/so/conf/zeek/local.zeek:/opt/zeek/share/zeek/site/local.zeek:ro \ + -v /opt/so/conf/zeek/node.cfg:/opt/zeek/etc/node.cfg:ro \ + -v /opt/so/conf/zeek/zeekctl.cfg:/opt/zeek/etc/zeekctl.cfg:ro \ + -v /opt/so/conf/zeek/policy/securityonion:/opt/zeek/share/zeek/policy/securityonion:ro \ + -v /opt/so/conf/zeek/policy/custom:/opt/zeek/share/zeek/policy/custom:ro \ + -v /opt/so/conf/zeek/policy/cve-2020-0601:/opt/zeek/share/zeek/policy/cve-2020-0601:ro \ + -v /opt/so/conf/zeek/policy/intel:/opt/zeek/share/zeek/policy/intel:rw \ + -v /opt/so/conf/zeek/bpf:/opt/zeek/etc/bpf:ro \ + --entrypoint /opt/zeek/bin/zeek \ + -w /nsm/zeek/logs \ + {{ MASTER }}:5000/soshybridhunter/so-zeek:{{ VERSION }} \ + -C -r /input.pcap local > $NSM_PATH/logs/console.log 2>&1 +} + +# if no parameters supplied, display usage +if [ $# -eq 0 ]; then + usage + exit +fi + +# ensure this is a manager node +if [ ! -d /opt/so/conf/soc ]; then + echo "This procedure must be run on a manager node." + exit +fi + +# verify that all parameters are files +for i in "$@"; do + if ! [ -f "$i" ]; then + usage + echo "\"$i\" is not a valid file!" + exit + fi +done + +# track if we have any valid or invalid pcaps +INVALID_PCAPS="no" +VALID_PCAPS="no" + +# track oldest start and newest end so that we can generate the Kibana search hyperlink at the end +START_OLDEST="2050-12-31" +END_NEWEST="1971-01-01" + +# paths must be quoted in case they include spaces +for PCAP in "$@"; do + PCAP=$(/usr/bin/realpath "$PCAP") + echo "Processing Import: ${PCAP}" + echo "- verifying file" + if ! pcapinfo "${PCAP}" > /dev/null 2>&1; then + # try to fix pcap and then process the fixed pcap directly + PCAP_FIXED=`mktemp /tmp/so-import-pcap-XXXXXXXXXX.pcap` + echo "- attempting to recover corrupted PCAP file" + pcapfix "${PCAP}" "${PCAP_FIXED}" + PCAP="${PCAP_FIXED}" + TEMP_PCAPS+=(${PCAP_FIXED}) + fi + + # generate a unique hash to assist with dedupe checks + HASH=$(md5sum "${PCAP}" | awk '{ print $1 }') + HASH_DIR=/nsm/import/${HASH} + echo "- assigning unique identifier to import: $HASH" + + if [ -d $HASH_DIR ]; then + echo "- this PCAP has already been imported; skipping" + INVALID_PCAPS="yes" + elif pcapinfo "${PCAP}" |egrep -q "Last packet time: 1970-01-01|Last packet time: n/a"; then + echo "- this PCAP file is invalid; skipping" + INVALID_PCAPS="yes" + else + VALID_PCAPS="yes" + + PCAP_DIR=$HASH_DIR/pcap + mkdir -p $PCAP_DIR + + # generate IDS alerts and write them to standard pipeline + echo "- analyzing traffic with Suricata" + suricata "${PCAP}" $HASH + + # generate Zeek logs and write them to a unique subdirectory in /nsm/import/bro/ + # since each run writes to a unique subdirectory, there is no need for a lock file + echo "- analyzing traffic with Zeek" + zeek "${PCAP}" $HASH + + START=$(pcapinfo "${PCAP}" -a |grep "First packet time:" | awk '{print $4}') + END=$(pcapinfo "${PCAP}" -e |grep "Last packet time:" | awk '{print $4}') + echo "- saving PCAP data spanning dates $START through $END" + + # compare $START to $START_OLDEST + START_COMPARE=$(date -d $START +%s) + START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s) + if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then + START_OLDEST=$START + fi + + # compare $ENDNEXT to $END_NEWEST + ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"` + ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s) + END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s) + if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then + END_NEWEST=$ENDNEXT + fi + + cp -f "${PCAP}" "${PCAP_DIR}"/data.pcap + + fi # end of valid pcap + + echo + +done # end of for-loop processing pcap files + +# remove temp files +echo "Cleaning up:" +for TEMP_PCAP in ${TEMP_PCAPS[@]}; do + echo "- removing temporary pcap $TEMP_PCAP" + rm -f $TEMP_PCAP +done + +# output final messages +if [ "$INVALID_PCAPS" = "yes" ]; then + echo + echo "Please note! One or more pcaps was invalid! You can scroll up to see which ones were invalid." +fi + +if [ "$VALID_PCAPS" = "yes" ]; then +cat << EOF + +Import complete! + +You can use the following hyperlink to view data in the time range of your import. You can triple-click to quickly highlight the entire hyperlink and you can then copy it into your browser: +https://{{ MASTERIP }}/kibana/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:'${START_OLDEST}T00:00:00.000Z',mode:absolute,to:'${END_NEWEST}T00:00:00.000Z')) + +or you can manually set your Time Range to be: +From: $START_OLDEST To: $END_NEWEST + +Please note that it may take 30 seconds or more for events to appear in Kibana. +EOF +fi \ No newline at end of file diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index da116cf2c..ac4e0fe6f 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -118,12 +118,31 @@ filebeat.inputs: clean_removed: false close_removed: false + - type: log + paths: + - /nsm/import/*/zeek/logs/{{ LOGNAME }}.log + fields: + module: zeek + dataset: {{ LOGNAME }} + category: network + imported: true + processors: + - dissect: + tokenizer: "/nsm/import/%{import_id}/zeek/logs/%{import_source}" + field: "source" + target_prefix: "" + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + + fields_under_root: true + clean_removed: false + close_removed: false {%- endfor %} {%- endif %} - type: log paths: - - /suricata/eve*.json + - /nsm/suricata/eve*.json fields: module: suricata dataset: common @@ -137,8 +156,27 @@ filebeat.inputs: clean_removed: false close_removed: false - {%- if STRELKAENABLED == 1 %} + - type: log + paths: + - /nsm/import/*/suricata/eve*.json + fields: + module: suricata + dataset: common + category: network + imported: true + processors: + - dissect: + tokenizer: "/nsm/import/%{import_id}/suricata/%{import_source}" + field: "source" + target_prefix: "" + - drop_fields: + fields: ["source", "prospector", "input", "offset", "beat"] + fields_under_root: true + clean_removed: false + close_removed: false + + {%- if STRELKAENABLED == 1 %} - type: log paths: - /nsm/strelka/log/strelka.log @@ -229,7 +267,7 @@ output.elasticsearch: - index: "so-strelka-%{+yyyy.MM.dd}" when.contains: module: "strelka" - + setup.template.enabled: false {%- else %} diff --git a/salt/filebeat/init.sls b/salt/filebeat/init.sls index 897bb3937..8fba7e258 100644 --- a/salt/filebeat/init.sls +++ b/salt/filebeat/init.sls @@ -55,9 +55,7 @@ so-filebeat: - binds: - /opt/so/log/filebeat:/usr/share/filebeat/logs:rw - /opt/so/conf/filebeat/etc/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - - /nsm/zeek:/nsm/zeek:ro - - /nsm/strelka/log:/nsm/strelka/log:ro - - /nsm/suricata:/suricata:ro + - /nsm:/nsm:ro - /opt/so/wazuh/logs/alerts:/wazuh/alerts:ro - /opt/so/wazuh/logs/archives:/wazuh/archives:ro - /nsm/osquery/fleet/:/nsm/osquery/fleet:ro From a3deb868ad6b5f1a8b1b7601feb084501c9dc4b8 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Tue, 30 Jun 2020 14:57:34 -0400 Subject: [PATCH 13/14] Improve filebeat config indentation --- salt/filebeat/etc/filebeat.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/salt/filebeat/etc/filebeat.yml b/salt/filebeat/etc/filebeat.yml index ac4e0fe6f..9dd00503c 100644 --- a/salt/filebeat/etc/filebeat.yml +++ b/salt/filebeat/etc/filebeat.yml @@ -128,9 +128,9 @@ filebeat.inputs: imported: true processors: - dissect: - tokenizer: "/nsm/import/%{import_id}/zeek/logs/%{import_source}" - field: "source" - target_prefix: "" + tokenizer: "/nsm/import/%{import_id}/zeek/logs/%{import_source}" + field: "source" + target_prefix: "" - drop_fields: fields: ["source", "prospector", "input", "offset", "beat"] @@ -166,9 +166,9 @@ filebeat.inputs: imported: true processors: - dissect: - tokenizer: "/nsm/import/%{import_id}/suricata/%{import_source}" - field: "source" - target_prefix: "" + tokenizer: "/nsm/import/%{import_id}/suricata/%{import_source}" + field: "source" + target_prefix: "" - drop_fields: fields: ["source", "prospector", "input", "offset", "beat"] From 7bb97f2b2d23c772137c34de7d960af1979c1a35 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Wed, 1 Jul 2020 11:12:21 -0400 Subject: [PATCH 14/14] [fix] Remove "Panel Title" from Kibana CPU graph Resolves #874 --- salt/grafana/dashboards/standalone/standalone.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/grafana/dashboards/standalone/standalone.json b/salt/grafana/dashboards/standalone/standalone.json index 7624e72f6..5b11eefb4 100644 --- a/salt/grafana/dashboards/standalone/standalone.json +++ b/salt/grafana/dashboards/standalone/standalone.json @@ -461,7 +461,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Panel Title{{ SERVERNAME }} - Kibana CPU", + "title": "{{ SERVERNAME }} - Kibana CPU", "tooltip": { "shared": true, "sort": 0,