mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2025-12-06 09:12:45 +01:00
221 lines
7.3 KiB
Bash
221 lines
7.3 KiB
Bash
#!/bin/bash
|
|
#
|
|
# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
{% set MASTER = salt['grains.get']('master') %}
|
|
{% set VERSION = salt['pillar.get']('static:soversion') %}
|
|
{%- set MASTERIP = salt['pillar.get']('static:masterip') -%}
|
|
|
|
function usage {
|
|
cat << EOF
|
|
Usage:
|
|
Please supply at least one pcap file.
|
|
|
|
For example, to import a single pcap named import.pcap:
|
|
so-import-pcap import.pcap
|
|
|
|
To import multiple pcaps:
|
|
so-import-pcap import1.pcap import2.pcap
|
|
|
|
** IMPORTANT **
|
|
Security Onion installations contain processes that automatically discard old data. Therefore, imports of old network traffic might immediately be erased, unless those processes are first disabled.
|
|
|
|
EOF
|
|
}
|
|
|
|
function pcapinfo() {
|
|
PCAP=$1
|
|
ARGS=$2
|
|
docker run --rm -v $PCAP:/input.pcap --entrypoint capinfos {{ MASTER }}:5000/soshybridhunter/so-pcaptools:{{ VERSION }} /input.pcap $ARGS
|
|
}
|
|
|
|
function pcapfix() {
|
|
PCAP=$1
|
|
PCAP_OUT=$2
|
|
docker run --rm -v $PCAP:/input.pcap -v $PCAP_OUT:$PCAP_OUT --entrypoint pcapfix {{ MASTER }}:5000/soshybridhunter/so-pcaptools:{{ VERSION }} /input.pcap -o $PCAP_OUT > /dev/null 2>&1
|
|
}
|
|
|
|
function suricata() {
|
|
PCAP=$1
|
|
HASH=$2
|
|
|
|
NSM_PATH=/nsm/import/${HASH}/suricata
|
|
mkdir -p $NSM_PATH
|
|
chown suricata:socore $NSM_PATH
|
|
LOG_PATH=/opt/so/log/suricata/import/${HASH}
|
|
mkdir -p $LOG_PATH
|
|
chown suricata:suricata $LOG_PATH
|
|
docker run --rm \
|
|
-v /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro \
|
|
-v /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro \
|
|
-v /opt/so/conf/suricata/rules:/etc/suricata/rules:ro \
|
|
-v ${LOG_PATH}:/var/log/suricata/:rw \
|
|
-v ${NSM_PATH}/:/nsm/:rw \
|
|
-v $PCAP:/input.pcap:ro \
|
|
-v /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro \
|
|
{{ MASTER }}:5000/soshybridhunter/so-suricata:{{ VERSION }} \
|
|
--runmode single -k none -r /input.pcap > $LOG_PATH/console.log 2>&1
|
|
}
|
|
|
|
function zeek() {
|
|
PCAP=$1
|
|
HASH=$2
|
|
|
|
NSM_PATH=/nsm/import/${HASH}/zeek
|
|
mkdir -p $NSM_PATH/logs
|
|
mkdir -p $NSM_PATH/spool
|
|
mkdir -p $NSM_PATH/extracted
|
|
chown -R zeek:socore $NSM_PATH
|
|
docker run --rm \
|
|
-v $NSM_PATH:/nsm:rw \
|
|
-v $PCAP:/input.pcap:ro \
|
|
-v /opt/so/conf/zeek/local.zeek:/opt/zeek/share/zeek/site/local.zeek:ro \
|
|
-v /opt/so/conf/zeek/node.cfg:/opt/zeek/etc/node.cfg:ro \
|
|
-v /opt/so/conf/zeek/zeekctl.cfg:/opt/zeek/etc/zeekctl.cfg:ro \
|
|
-v /opt/so/conf/zeek/policy/securityonion:/opt/zeek/share/zeek/policy/securityonion:ro \
|
|
-v /opt/so/conf/zeek/policy/custom:/opt/zeek/share/zeek/policy/custom:ro \
|
|
-v /opt/so/conf/zeek/policy/cve-2020-0601:/opt/zeek/share/zeek/policy/cve-2020-0601:ro \
|
|
-v /opt/so/conf/zeek/policy/intel:/opt/zeek/share/zeek/policy/intel:rw \
|
|
-v /opt/so/conf/zeek/bpf:/opt/zeek/etc/bpf:ro \
|
|
--entrypoint /opt/zeek/bin/zeek \
|
|
-w /nsm/zeek/logs \
|
|
{{ MASTER }}:5000/soshybridhunter/so-zeek:{{ VERSION }} \
|
|
-C -r /input.pcap local > $NSM_PATH/logs/console.log 2>&1
|
|
}
|
|
|
|
# if no parameters supplied, display usage
|
|
if [ $# -eq 0 ]; then
|
|
usage
|
|
exit
|
|
fi
|
|
|
|
# ensure this is a manager node
|
|
if [ ! -d /opt/so/conf/soc ]; then
|
|
echo "This procedure must be run on a manager node."
|
|
exit
|
|
fi
|
|
|
|
# verify that all parameters are files
|
|
for i in "$@"; do
|
|
if ! [ -f "$i" ]; then
|
|
usage
|
|
echo "\"$i\" is not a valid file!"
|
|
exit
|
|
fi
|
|
done
|
|
|
|
# track if we have any valid or invalid pcaps
|
|
INVALID_PCAPS="no"
|
|
VALID_PCAPS="no"
|
|
|
|
# track oldest start and newest end so that we can generate the Kibana search hyperlink at the end
|
|
START_OLDEST="2050-12-31"
|
|
END_NEWEST="1971-01-01"
|
|
|
|
# paths must be quoted in case they include spaces
|
|
for PCAP in "$@"; do
|
|
PCAP=$(/usr/bin/realpath "$PCAP")
|
|
echo "Processing Import: ${PCAP}"
|
|
echo "- verifying file"
|
|
if ! pcapinfo "${PCAP}" > /dev/null 2>&1; then
|
|
# try to fix pcap and then process the fixed pcap directly
|
|
PCAP_FIXED=`mktemp /tmp/so-import-pcap-XXXXXXXXXX.pcap`
|
|
echo "- attempting to recover corrupted PCAP file"
|
|
pcapfix "${PCAP}" "${PCAP_FIXED}"
|
|
PCAP="${PCAP_FIXED}"
|
|
TEMP_PCAPS+=(${PCAP_FIXED})
|
|
fi
|
|
|
|
# generate a unique hash to assist with dedupe checks
|
|
HASH=$(md5sum "${PCAP}" | awk '{ print $1 }')
|
|
HASH_DIR=/nsm/import/${HASH}
|
|
echo "- assigning unique identifier to import: $HASH"
|
|
|
|
if [ -d $HASH_DIR ]; then
|
|
echo "- this PCAP has already been imported; skipping"
|
|
INVALID_PCAPS="yes"
|
|
elif pcapinfo "${PCAP}" |egrep -q "Last packet time: 1970-01-01|Last packet time: n/a"; then
|
|
echo "- this PCAP file is invalid; skipping"
|
|
INVALID_PCAPS="yes"
|
|
else
|
|
VALID_PCAPS="yes"
|
|
|
|
PCAP_DIR=$HASH_DIR/pcap
|
|
mkdir -p $PCAP_DIR
|
|
|
|
# generate IDS alerts and write them to standard pipeline
|
|
echo "- analyzing traffic with Suricata"
|
|
suricata "${PCAP}" $HASH
|
|
|
|
# generate Zeek logs and write them to a unique subdirectory in /nsm/import/bro/
|
|
# since each run writes to a unique subdirectory, there is no need for a lock file
|
|
echo "- analyzing traffic with Zeek"
|
|
zeek "${PCAP}" $HASH
|
|
|
|
START=$(pcapinfo "${PCAP}" -a |grep "First packet time:" | awk '{print $4}')
|
|
END=$(pcapinfo "${PCAP}" -e |grep "Last packet time:" | awk '{print $4}')
|
|
echo "- saving PCAP data spanning dates $START through $END"
|
|
|
|
# compare $START to $START_OLDEST
|
|
START_COMPARE=$(date -d $START +%s)
|
|
START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s)
|
|
if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then
|
|
START_OLDEST=$START
|
|
fi
|
|
|
|
# compare $ENDNEXT to $END_NEWEST
|
|
ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"`
|
|
ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s)
|
|
END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s)
|
|
if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then
|
|
END_NEWEST=$ENDNEXT
|
|
fi
|
|
|
|
cp -f "${PCAP}" "${PCAP_DIR}"/data.pcap
|
|
|
|
fi # end of valid pcap
|
|
|
|
echo
|
|
|
|
done # end of for-loop processing pcap files
|
|
|
|
# remove temp files
|
|
echo "Cleaning up:"
|
|
for TEMP_PCAP in ${TEMP_PCAPS[@]}; do
|
|
echo "- removing temporary pcap $TEMP_PCAP"
|
|
rm -f $TEMP_PCAP
|
|
done
|
|
|
|
# output final messages
|
|
if [ "$INVALID_PCAPS" = "yes" ]; then
|
|
echo
|
|
echo "Please note! One or more pcaps was invalid! You can scroll up to see which ones were invalid."
|
|
fi
|
|
|
|
if [ "$VALID_PCAPS" = "yes" ]; then
|
|
cat << EOF
|
|
|
|
Import complete!
|
|
|
|
You can use the following hyperlink to view data in the time range of your import. You can triple-click to quickly highlight the entire hyperlink and you can then copy it into your browser:
|
|
https://{{ MASTERIP }}/kibana/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:'${START_OLDEST}T00:00:00.000Z',mode:absolute,to:'${END_NEWEST}T00:00:00.000Z'))
|
|
|
|
or you can manually set your Time Range to be:
|
|
From: $START_OLDEST To: $END_NEWEST
|
|
|
|
Please note that it may take 30 seconds or more for events to appear in Kibana.
|
|
EOF
|
|
fi |