mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2025-12-20 16:03:06 +01:00
214 lines
7.3 KiB
Bash
Executable File
214 lines
7.3 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Copyright 2014,2015,2016,2017,2018,2019,2020 Security Onion Solutions, LLC
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
{% set MASTER = salt['grains.get']('master') %}
|
|
{% set VERSION = salt['pillar.get']('static:soversion') %}
|
|
{%- set MASTERIP = salt['pillar.get']('static:masterip') -%}
|
|
|
|
function usage {
|
|
cat << EOF
|
|
Usage: $0 <pcap-file-1> [pcap-file-2] [pcap-file-3]
|
|
|
|
Imports the given file(s) into the Security Onion system. Be aware that importing PCAP files with traffic dated older than the curator threshold will result in missing data. Use the included "so-curator-stop" command to avoid this scenario.
|
|
EOF
|
|
}
|
|
|
|
function pcapinfo() {
|
|
PCAP=$1
|
|
ARGS=$2
|
|
docker run --rm -v $PCAP:/input.pcap --entrypoint capinfos {{ MASTER }}:5000/soshybridhunter/so-pcaptools:{{ VERSION }} /input.pcap $ARGS
|
|
}
|
|
|
|
function pcapfix() {
|
|
PCAP=$1
|
|
PCAP_OUT=$2
|
|
docker run --rm -v $PCAP:/input.pcap -v $PCAP_OUT:$PCAP_OUT --entrypoint pcapfix {{ MASTER }}:5000/soshybridhunter/so-pcaptools:{{ VERSION }} /input.pcap -o $PCAP_OUT > /dev/null 2>&1
|
|
}
|
|
|
|
function suricata() {
|
|
PCAP=$1
|
|
HASH=$2
|
|
|
|
NSM_PATH=/nsm/import/${HASH}/suricata
|
|
mkdir -p $NSM_PATH
|
|
chown suricata:socore $NSM_PATH
|
|
LOG_PATH=/opt/so/log/suricata/import/${HASH}
|
|
mkdir -p $LOG_PATH
|
|
chown suricata:suricata $LOG_PATH
|
|
docker run --rm \
|
|
-v /opt/so/conf/suricata/suricata.yaml:/etc/suricata/suricata.yaml:ro \
|
|
-v /opt/so/conf/suricata/threshold.conf:/etc/suricata/threshold.conf:ro \
|
|
-v /opt/so/conf/suricata/rules:/etc/suricata/rules:ro \
|
|
-v ${LOG_PATH}:/var/log/suricata/:rw \
|
|
-v ${NSM_PATH}/:/nsm/:rw \
|
|
-v $PCAP:/input.pcap:ro \
|
|
-v /opt/so/conf/suricata/bpf:/etc/suricata/bpf:ro \
|
|
{{ MASTER }}:5000/soshybridhunter/so-suricata:{{ VERSION }} \
|
|
--runmode single -k none -r /input.pcap > $LOG_PATH/console.log 2>&1
|
|
}
|
|
|
|
function zeek() {
|
|
PCAP=$1
|
|
HASH=$2
|
|
|
|
NSM_PATH=/nsm/import/${HASH}/zeek
|
|
mkdir -p $NSM_PATH/logs
|
|
mkdir -p $NSM_PATH/extracted
|
|
mkdir -p $NSM_PATH/spool
|
|
chown -R zeek:socore $NSM_PATH
|
|
docker run --rm \
|
|
-v $NSM_PATH/logs:/nsm/zeek/logs:rw \
|
|
-v $NSM_PATH/spool:/nsm/zeek/spool:rw \
|
|
-v $NSM_PATH/extracted:/nsm/zeek/extracted:rw \
|
|
-v $PCAP:/input.pcap:ro \
|
|
-v /opt/so/conf/zeek/local.zeek:/opt/zeek/share/zeek/site/local.zeek:ro \
|
|
-v /opt/so/conf/zeek/node.cfg:/opt/zeek/etc/node.cfg:ro \
|
|
-v /opt/so/conf/zeek/zeekctl.cfg:/opt/zeek/etc/zeekctl.cfg:ro \
|
|
-v /opt/so/conf/zeek/policy/securityonion:/opt/zeek/share/zeek/policy/securityonion:ro \
|
|
-v /opt/so/conf/zeek/policy/custom:/opt/zeek/share/zeek/policy/custom:ro \
|
|
-v /opt/so/conf/zeek/policy/cve-2020-0601:/opt/zeek/share/zeek/policy/cve-2020-0601:ro \
|
|
-v /opt/so/conf/zeek/policy/intel:/opt/zeek/share/zeek/policy/intel:rw \
|
|
-v /opt/so/conf/zeek/bpf:/opt/zeek/etc/bpf:ro \
|
|
--entrypoint /opt/zeek/bin/zeek \
|
|
-w /nsm/zeek/logs \
|
|
{{ MASTER }}:5000/soshybridhunter/so-zeek:{{ VERSION }} \
|
|
-C -r /input.pcap local > $NSM_PATH/logs/console.log 2>&1
|
|
}
|
|
|
|
# if no parameters supplied, display usage
|
|
if [ $# -eq 0 ]; then
|
|
usage
|
|
exit
|
|
fi
|
|
|
|
# ensure this is a manager node
|
|
if [ ! -d /opt/so/conf/soc ]; then
|
|
echo "This procedure must be run on a manager node."
|
|
exit
|
|
fi
|
|
|
|
# verify that all parameters are files
|
|
for i in "$@"; do
|
|
if ! [ -f "$i" ]; then
|
|
usage
|
|
echo "\"$i\" is not a valid file!"
|
|
exit
|
|
fi
|
|
done
|
|
|
|
# track if we have any valid or invalid pcaps
|
|
INVALID_PCAPS="no"
|
|
VALID_PCAPS="no"
|
|
|
|
# track oldest start and newest end so that we can generate the Kibana search hyperlink at the end
|
|
START_OLDEST="2050-12-31"
|
|
END_NEWEST="1971-01-01"
|
|
|
|
# paths must be quoted in case they include spaces
|
|
for PCAP in "$@"; do
|
|
PCAP=$(/usr/bin/realpath "$PCAP")
|
|
echo "Processing Import: ${PCAP}"
|
|
echo "- verifying file"
|
|
if ! pcapinfo "${PCAP}" > /dev/null 2>&1; then
|
|
# try to fix pcap and then process the fixed pcap directly
|
|
PCAP_FIXED=`mktemp /tmp/so-import-pcap-XXXXXXXXXX.pcap`
|
|
echo "- attempting to recover corrupted PCAP file"
|
|
pcapfix "${PCAP}" "${PCAP_FIXED}"
|
|
PCAP="${PCAP_FIXED}"
|
|
TEMP_PCAPS+=(${PCAP_FIXED})
|
|
fi
|
|
|
|
# generate a unique hash to assist with dedupe checks
|
|
HASH=$(md5sum "${PCAP}" | awk '{ print $1 }')
|
|
HASH_DIR=/nsm/import/${HASH}
|
|
echo "- assigning unique identifier to import: $HASH"
|
|
|
|
if [ -d $HASH_DIR ]; then
|
|
echo "- this PCAP has already been imported; skipping"
|
|
INVALID_PCAPS="yes"
|
|
elif pcapinfo "${PCAP}" |egrep -q "Last packet time: 1970-01-01|Last packet time: n/a"; then
|
|
echo "- this PCAP file is invalid; skipping"
|
|
INVALID_PCAPS="yes"
|
|
else
|
|
VALID_PCAPS="yes"
|
|
|
|
PCAP_DIR=$HASH_DIR/pcap
|
|
mkdir -p $PCAP_DIR
|
|
|
|
# generate IDS alerts and write them to standard pipeline
|
|
echo "- analyzing traffic with Suricata"
|
|
suricata "${PCAP}" $HASH
|
|
|
|
# generate Zeek logs and write them to a unique subdirectory in /nsm/import/bro/
|
|
# since each run writes to a unique subdirectory, there is no need for a lock file
|
|
echo "- analyzing traffic with Zeek"
|
|
zeek "${PCAP}" $HASH
|
|
|
|
START=$(pcapinfo "${PCAP}" -a |grep "First packet time:" | awk '{print $4}')
|
|
END=$(pcapinfo "${PCAP}" -e |grep "Last packet time:" | awk '{print $4}')
|
|
echo "- saving PCAP data spanning dates $START through $END"
|
|
|
|
# compare $START to $START_OLDEST
|
|
START_COMPARE=$(date -d $START +%s)
|
|
START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s)
|
|
if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then
|
|
START_OLDEST=$START
|
|
fi
|
|
|
|
# compare $ENDNEXT to $END_NEWEST
|
|
ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"`
|
|
ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s)
|
|
END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s)
|
|
if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then
|
|
END_NEWEST=$ENDNEXT
|
|
fi
|
|
|
|
cp -f "${PCAP}" "${PCAP_DIR}"/data.pcap
|
|
|
|
fi # end of valid pcap
|
|
|
|
echo
|
|
|
|
done # end of for-loop processing pcap files
|
|
|
|
# remove temp files
|
|
echo "Cleaning up:"
|
|
for TEMP_PCAP in ${TEMP_PCAPS[@]}; do
|
|
echo "- removing temporary pcap $TEMP_PCAP"
|
|
rm -f $TEMP_PCAP
|
|
done
|
|
|
|
# output final messages
|
|
if [ "$INVALID_PCAPS" = "yes" ]; then
|
|
echo
|
|
echo "Please note! One or more pcaps was invalid! You can scroll up to see which ones were invalid."
|
|
fi
|
|
|
|
if [ "$VALID_PCAPS" = "yes" ]; then
|
|
cat << EOF
|
|
|
|
Import complete!
|
|
|
|
You can use the following hyperlink to view data in the time range of your import. You can triple-click to quickly highlight the entire hyperlink and you can then copy it into your browser:
|
|
https://{{ MASTERIP }}/kibana/app/kibana#/dashboard/a8411b30-6d03-11ea-b301-3d6c35840645?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:'${START_OLDEST}T00:00:00.000Z',mode:absolute,to:'${END_NEWEST}T00:00:00.000Z'))
|
|
|
|
or you can manually set your Time Range to be:
|
|
From: $START_OLDEST To: $END_NEWEST
|
|
|
|
Please note that it may take 30 seconds or more for events to appear in Kibana.
|
|
EOF
|
|
fi |