Merge pull request #10940 from Security-Onion-Solutions/foxtrot

Add time shift for so-import-evtx
This commit is contained in:
weslambert
2023-08-03 16:56:40 -04:00
committed by GitHub

View File

@@ -27,6 +27,8 @@ Imports one or more evtx files into Security Onion. The evtx files will be analy
Options: Options:
--json Outputs summary in JSON format. Implies --quiet. --json Outputs summary in JSON format. Implies --quiet.
--quiet Silences progress information to stdout. --quiet Silences progress information to stdout.
--shift Adds a time shift. Accepts a single argument that is intended to be the date of the last record, and shifts the dates of the previous records accordingly.
Ex. sudo so-import-evtx --shift "2023-08-01 01:01:01" example.evtx
EOF EOF
} }
@@ -44,6 +46,10 @@ while [[ $# -gt 0 ]]; do
--quiet) --quiet)
quiet=1 quiet=1
;; ;;
--shift)
SHIFTDATE=$1
shift
;;
-*) -*)
echo "Encountered unexpected parameter: $param" echo "Encountered unexpected parameter: $param"
usage usage
@@ -68,8 +74,10 @@ function status {
function evtx2es() { function evtx2es() {
EVTX=$1 EVTX=$1
HASH=$2 HASH=$2
SHIFTDATE=$3
docker run --rm \ docker run --rm \
-e "SHIFTTS=$SHIFTDATE" \
-v "$EVTX:/tmp/data.evtx" \ -v "$EVTX:/tmp/data.evtx" \
-v "/nsm/import/$HASH/evtx/:/tmp/evtx/" \ -v "/nsm/import/$HASH/evtx/:/tmp/evtx/" \
-v "/nsm/import/evtx-end_newest:/tmp/newest" \ -v "/nsm/import/evtx-end_newest:/tmp/newest" \
@@ -113,7 +121,9 @@ echo $END_NEWEST > /nsm/import/evtx-end_newest
for EVTX in $INPUT_FILES; do for EVTX in $INPUT_FILES; do
EVTX=$(/usr/bin/realpath "$EVTX") EVTX=$(/usr/bin/realpath "$EVTX")
status "Processing Import: ${EVTX}" status "Processing Import: ${EVTX}"
if ! [ -z "$SHIFTDATE" ]; then
status "- timeshifting logs to end date of $SHIFTDATE"
fi
# generate a unique hash to assist with dedupe checks # generate a unique hash to assist with dedupe checks
HASH=$(md5sum "${EVTX}" | awk '{ print $1 }') HASH=$(md5sum "${EVTX}" | awk '{ print $1 }')
HASH_DIR=/nsm/import/${HASH} HASH_DIR=/nsm/import/${HASH}
@@ -136,7 +146,7 @@ for EVTX in $INPUT_FILES; do
# import evtx and write them to import ingest pipeline # import evtx and write them to import ingest pipeline
status "- importing logs to Elasticsearch..." status "- importing logs to Elasticsearch..."
evtx2es "${EVTX}" $HASH evtx2es "${EVTX}" $HASH "$SHIFTDATE"
if [[ $? -ne 0 ]]; then if [[ $? -ne 0 ]]; then
INVALID_EVTXS_COUNT=$((INVALID_EVTXS_COUNT + 1)) INVALID_EVTXS_COUNT=$((INVALID_EVTXS_COUNT + 1))
status "- WARNING: This evtx file may not have fully imported successfully" status "- WARNING: This evtx file may not have fully imported successfully"
@@ -222,4 +232,4 @@ if [[ $json -eq 1 ]]; then
}''' }'''
fi fi
exit $RESULT exit $RESULT