diff --git a/salt/common/tools/sbin/so-import-evtx b/salt/common/tools/sbin/so-import-evtx
new file mode 100644
index 000000000..9e640beaa
--- /dev/null
+++ b/salt/common/tools/sbin/so-import-evtx
@@ -0,0 +1,172 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+{%- set MANAGER = salt['grains.get']('master') %}
+{%- set VERSION = salt['pillar.get']('global:soversion') %}
+{%- set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
+{%- set MANAGERIP = salt['pillar.get']('global:managerip') -%}
+{%- set URLBASE = salt['pillar.get']('global:url_base') %}
+{% set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %}
+{% set ES_PW = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %}
+
+INDEX_DATE=$(date +'%Y.%m.%d')
+RUNID=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1)
+
+. /usr/sbin/so-common
+
+function usage {
+ cat << EOF
+Usage: $0 [evtx-file-2] [evtx-file-*]
+
+Imports one or more evtx files into Security Onion. The evtx files will be analyzed and made available for review in the Security Onion toolset.
+EOF
+}
+
+
+function evtx2es() {
+ EVTX=$1
+ HASH=$2
+
+ docker run --rm \
+ -v "$EVTX:/tmp/$RUNID.evtx" \
+ --entrypoint evtx2es \
+ {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} \
+ --host {{ MANAGERIP }} --scheme https \
+ --index so-beats-$INDEX_DATE --pipeline import.wel \
+ --login {{ES_USER}} --pwd {{ES_PW}} \
+ "/tmp/$RUNID.evtx" 1>/dev/null 2>/dev/null
+
+ docker run --rm \
+ -v "$EVTX:/tmp/import.evtx" \
+ -v "/nsm/import/evtx-end_newest:/tmp/newest" \
+ -v "/nsm/import/evtx-start_oldest:/tmp/oldest" \
+ --entrypoint '/evtx_calc_timestamps.sh' \
+ {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }}
+}
+
+# if no parameters supplied, display usage
+if [ $# -eq 0 ]; then
+ usage
+ exit 1
+fi
+
+# ensure this is a Manager node
+require_manager
+
+# verify that all parameters are files
+for i in "$@"; do
+ if ! [ -f "$i" ]; then
+ usage
+ echo "\"$i\" is not a valid file!"
+ exit 2
+ fi
+done
+
+# track if we have any valid or invalid evtx
+INVALID_EVTXS="no"
+VALID_EVTXS="no"
+
+# track oldest start and newest end so that we can generate the Kibana search hyperlink at the end
+START_OLDEST="2050-12-31"
+END_NEWEST="1971-01-01"
+
+touch /nsm/import/evtx-start_oldest
+touch /nsm/import/evtx-end_newest
+
+echo $START_OLDEST > /nsm/import/evtx-start_oldest
+echo $END_NEWEST > /nsm/import/evtx-end_newest
+
+# paths must be quoted in case they include spaces
+for EVTX in "$@"; do
+ EVTX=$(/usr/bin/realpath "$EVTX")
+ echo "Processing Import: ${EVTX}"
+
+ # generate a unique hash to assist with dedupe checks
+ HASH=$(md5sum "${EVTX}" | awk '{ print $1 }')
+ HASH_DIR=/nsm/import/${HASH}
+ echo "- assigning unique identifier to import: $HASH"
+
+ if [ -d $HASH_DIR ]; then
+ echo "- this EVTX has already been imported; skipping"
+ INVALID_EVTXS="yes"
+ else
+ VALID_EVTXS="yes"
+
+ EVTX_DIR=$HASH_DIR/evtx
+ mkdir -p $EVTX_DIR
+
+ # import evtx and write them to import ingest pipeline
+ echo "- importing logs to Elasticsearch..."
+ evtx2es "${EVTX}" $HASH
+
+ # compare $START to $START_OLDEST
+ START=$(cat /nsm/import/evtx-start_oldest)
+ START_COMPARE=$(date -d $START +%s)
+ START_OLDEST_COMPARE=$(date -d $START_OLDEST +%s)
+ if [ $START_COMPARE -lt $START_OLDEST_COMPARE ]; then
+ START_OLDEST=$START
+ fi
+
+ # compare $ENDNEXT to $END_NEWEST
+ END=$(cat /nsm/import/evtx-end_newest)
+ ENDNEXT=`date +%Y-%m-%d --date="$END 1 day"`
+ ENDNEXT_COMPARE=$(date -d $ENDNEXT +%s)
+ END_NEWEST_COMPARE=$(date -d $END_NEWEST +%s)
+ if [ $ENDNEXT_COMPARE -gt $END_NEWEST_COMPARE ]; then
+ END_NEWEST=$ENDNEXT
+ fi
+
+ cp -f "${EVTX}" "${EVTX_DIR}"/data.evtx
+ chmod 644 "${EVTX_DIR}"/data.evtx
+
+ fi # end of valid evtx
+
+ echo
+
+done # end of for-loop processing evtx files
+
+# remove temp files
+echo "Cleaning up:"
+for TEMP_EVTX in ${TEMP_EVTXS[@]}; do
+ echo "- removing temporary evtx $TEMP_EVTX"
+ rm -f $TEMP_EVTX
+done
+
+# output final messages
+if [ "$INVALID_EVTXS" = "yes" ]; then
+ echo
+ echo "Please note! One or more evtx was invalid! You can scroll up to see which ones were invalid."
+fi
+
+START_OLDEST_FORMATTED=`date +%Y-%m-%d --date="$START_OLDEST"`
+START_OLDEST_SLASH=$(echo $START_OLDEST_FORMATTED | sed -e 's/-/%2F/g')
+END_NEWEST_SLASH=$(echo $END_NEWEST | sed -e 's/-/%2F/g')
+
+if [ "$VALID_EVTXS" = "yes" ]; then
+cat << EOF
+
+Import complete!
+
+You can use the following hyperlink to view data in the time range of your import. You can triple-click to quickly highlight the entire hyperlink and you can then copy it into your browser:
+https://{{ URLBASE }}/#/hunt?q=import.id:${RUNID}%20%7C%20groupby%20event.module%20event.dataset&t=${START_OLDEST_SLASH}%2000%3A00%3A00%20AM%20-%20${END_NEWEST_SLASH}%2000%3A00%3A00%20AM&z=UTC
+
+or you can manually set your Time Range to be (in UTC):
+From: $START_OLDEST_FORMATTED To: $END_NEWEST
+
+Please note that it may take 30 seconds or more for events to appear in Hunt.
+EOF
+fi
diff --git a/salt/common/tools/sbin/so-yara-update b/salt/common/tools/sbin/so-yara-update
index 9f749727f..025cf8c70 100755
--- a/salt/common/tools/sbin/so-yara-update
+++ b/salt/common/tools/sbin/so-yara-update
@@ -1,5 +1,4 @@
#!/bin/bash
-
# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
#
# This program is free software: you can redistribute it and/or modify
@@ -20,13 +19,8 @@ echo "Starting to check for yara rule updates at $(date)..."
output_dir="/opt/so/saltstack/default/salt/strelka/rules"
mkdir -p $output_dir
-
repos="$output_dir/repos.txt"
-ignorefile="$output_dir/ignore.txt"
-
-deletecounter=0
newcounter=0
-updatecounter=0
{% if ISAIRGAP is sameas true %}
@@ -35,58 +29,21 @@ echo "Airgap mode enabled."
clone_dir="/nsm/repo/rules/strelka"
repo_name="signature-base"
mkdir -p /opt/so/saltstack/default/salt/strelka/rules/signature-base
-
+# Ensure a copy of the license is available for the rules
[ -f $clone_dir/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name
# Copy over rules
for i in $(find $clone_dir/yara -name "*.yar*"); do
rule_name=$(echo $i | awk -F '/' '{print $NF}')
- repo_sum=$(sha256sum $i | awk '{print $1}')
-
- # Check rules against those in ignore list -- don't copy if ignored.
- if ! grep -iq $rule_name $ignorefile; then
- existing_rules=$(find $output_dir/$repo_name/ -name $rule_name | wc -l)
-
- # For existing rules, check to see if they need to be updated, by comparing checksums
- if [ $existing_rules -gt 0 ];then
- local_sum=$(sha256sum $output_dir/$repo_name/$rule_name | awk '{print $1}')
- if [ "$repo_sum" != "$local_sum" ]; then
- echo "Checksums do not match!"
- echo "Updating $rule_name..."
- cp $i $output_dir/$repo_name;
- ((updatecounter++))
- fi
- else
- # If rule doesn't exist already, we'll add it
- echo "Adding new rule: $rule_name..."
- cp $i $output_dir/$repo_name
- ((newcounter++))
- fi
- fi;
-done
-
-# Check to see if we have any old rules that need to be removed
-for i in $(find $output_dir/$repo_name -name "*.yar*" | awk -F '/' '{print $NF}'); do
- is_repo_rule=$(find $clone_dir -name "$i" | wc -l)
- if [ $is_repo_rule -eq 0 ]; then
- echo "Could not find $i in source $repo_name repo...removing from $output_dir/$repo_name..."
- rm $output_dir/$repo_name/$i
- ((deletecounter++))
- fi
+ echo "Adding rule: $rule_name..."
+ cp $i $output_dir/$repo_name
+ ((newcounter++))
done
echo "Done!"
if [ "$newcounter" -gt 0 ];then
- echo "$newcounter new rules added."
-fi
-
-if [ "$updatecounter" -gt 0 ];then
- echo "$updatecounter rules updated."
-fi
-
-if [ "$deletecounter" -gt 0 ];then
- echo "$deletecounter rules removed because they were deprecated or don't exist in the source repo."
+ echo "$newcounter rules added."
fi
{% else %}
@@ -99,69 +56,32 @@ if [ "$gh_status" == "200" ] || [ "$gh_status" == "301" ]; then
if ! $(echo "$repo" | grep -qE '^#'); then
# Remove old repo if existing bc of previous error condition or unexpected disruption
repo_name=`echo $repo | awk -F '/' '{print $NF}'`
- [ -d $repo_name ] && rm -rf $repo_name
+ [ -d $output_dir/$repo_name ] && rm -rf $output_dir/$repo_name
# Clone repo and make appropriate directories for rules
-
git clone $repo $clone_dir/$repo_name
echo "Analyzing rules from $clone_dir/$repo_name..."
mkdir -p $output_dir/$repo_name
+ # Ensure a copy of the license is available for the rules
[ -f $clone_dir/$repo_name/LICENSE ] && cp $clone_dir/$repo_name/LICENSE $output_dir/$repo_name
# Copy over rules
for i in $(find $clone_dir/$repo_name -name "*.yar*"); do
rule_name=$(echo $i | awk -F '/' '{print $NF}')
- repo_sum=$(sha256sum $i | awk '{print $1}')
-
- # Check rules against those in ignore list -- don't copy if ignored.
- if ! grep -iq $rule_name $ignorefile; then
- existing_rules=$(find $output_dir/$repo_name/ -name $rule_name | wc -l)
-
- # For existing rules, check to see if they need to be updated, by comparing checksums
- if [ $existing_rules -gt 0 ];then
- local_sum=$(sha256sum $output_dir/$repo_name/$rule_name | awk '{print $1}')
- if [ "$repo_sum" != "$local_sum" ]; then
- echo "Checksums do not match!"
- echo "Updating $rule_name..."
- cp $i $output_dir/$repo_name;
- ((updatecounter++))
- fi
- else
- # If rule doesn't exist already, we'll add it
- echo "Adding new rule: $rule_name..."
- cp $i $output_dir/$repo_name
- ((newcounter++))
- fi
- fi;
- done
-
- # Check to see if we have any old rules that need to be removed
- for i in $(find $output_dir/$repo_name -name "*.yar*" | awk -F '/' '{print $NF}'); do
- is_repo_rule=$(find $clone_dir/$repo_name -name "$i" | wc -l)
- if [ $is_repo_rule -eq 0 ]; then
- echo "Could not find $i in source $repo_name repo...removing from $output_dir/$repo_name..."
- rm $output_dir/$repo_name/$i
- ((deletecounter++))
- fi
- done
- rm -rf $clone_dir/$repo_name
- fi
- done < $repos
+ echo "Adding rule: $rule_name..."
+ cp $i $output_dir/$repo_name
+ ((newcounter++))
+ done
+ rm -rf $clone_dir/$repo_name
+ fi
+ done < $repos
echo "Done!"
-
+
if [ "$newcounter" -gt 0 ];then
- echo "$newcounter new rules added."
+ echo "$newcounter rules added."
fi
-
- if [ "$updatecounter" -gt 0 ];then
- echo "$updatecounter rules updated."
- fi
-
- if [ "$deletecounter" -gt 0 ];then
- echo "$deletecounter rules removed because they were deprecated or don't exist in the source repo."
- fi
-
+
else
echo "Server returned $gh_status status code."
echo "No connectivity to Github...exiting..."
diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup
index 17339fdd5..c8935969d 100755
--- a/salt/common/tools/sbin/soup
+++ b/salt/common/tools/sbin/soup
@@ -385,6 +385,7 @@ preupgrade_changes() {
[[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20
[[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
[[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50
+ [[ "$INSTALLEDVERSION" == 2.3.50 || "$INSTALLEDVERSION" == 2.3.51 || "$INSTALLEDVERSION" == 2.3.52 || "$INSTALLEDVERSION" == 2.3.60 || "$INSTALLEDVERSION" == 2.3.61 || "$INSTALLEDVERSION" == 2.3.70]] && up_2.3.5X_to_2.3.80
true
}
@@ -622,6 +623,20 @@ EOF
INSTALLEDVERSION=2.3.50
}
+up_2.3.5X_to_2.3.80() {
+
+ # Remove watermark settings from global.sls
+ sed -i '/ cluster_routing_allocation_disk/d' /opt/so/saltstack/local/pillar/global.sls
+
+ # Add new indices to the global
+ sed -i '/ index_settings:/a \\ so-elasticsearch: \n shards: 1 \n warm: 7 \n close: 30 \n delete: 365' /opt/so/saltstack/local/pillar/global.sls
+ sed -i '/ index_settings:/a \\ so-logstash: \n shards: 1 \n warm: 7 \n close: 30 \n delete: 365' /opt/so/saltstack/local/pillar/global.sls
+ sed -i '/ index_settings:/a \\ so-kibana: \n shards: 1 \n warm: 7 \n close: 30 \n delete: 365' /opt/so/saltstack/local/pillar/global.sls
+ sed -i '/ index_settings:/a \\ so-redis: \n shards: 1 \n warm: 7 \n close: 30 \n delete: 365' /opt/so/saltstack/local/pillar/global.sls
+
+ INSTALLEDVERSION=2.3.80
+}
+
verify_upgradespace() {
CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
if [ "$CURRENTSPACE" -lt "10" ]; then
diff --git a/salt/elasticsearch/files/ingest/import.wel b/salt/elasticsearch/files/ingest/import.wel
index 5a04324b7..e75098f8f 100644
--- a/salt/elasticsearch/files/ingest/import.wel
+++ b/salt/elasticsearch/files/ingest/import.wel
@@ -4,6 +4,7 @@
{ "remove": { "field": ["event.created","timestamp", "winlog.event_data.UtcTime", "event_record_id"], "ignore_failure": true } },
{ "pipeline": { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'", "name": "sysmon" } },
{ "pipeline": { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'", "name":"win.eventlogs" } },
+ { "dissect": { "field": "log.file.name", "pattern" : "/tmp/%{import.id}.evtx" } },
{ "pipeline": { "name": "common" } }
]
}
\ No newline at end of file
diff --git a/salt/soc/files/soc/soc.json b/salt/soc/files/soc/soc.json
index 6119f0e6b..e9dfa50d8 100644
--- a/salt/soc/files/soc/soc.json
+++ b/salt/soc/files/soc/soc.json
@@ -3,7 +3,8 @@
{%- set THEHIVEKEY = salt['pillar.get']('global:hivekey', '') %}
{%- set PLAYBOOK = salt['pillar.get']('manager:playbook', '0') %}
{%- set THEHIVE = salt['pillar.get']('manager:thehive', '0') %}
-{%- set OSQUERY = salt['pillar.get']('manager:osquery', '0') %}
+{% set FLEETMANAGER = salt['pillar.get']('global:fleet_manager', False) %}
+{% set FLEETNODE = salt['pillar.get']('global:fleet_node', False) %}
{%- set GRAFANA = salt['pillar.get']('manager:grafana', '0') %}
{%- set ISAIRGAP = salt['pillar.get']('global:airgap', 'False') %}
{%- set API_TIMEOUT = salt['pillar.get']('sensoroni:api_timeout_ms', 0) %}
@@ -116,7 +117,7 @@
{%- if THEHIVE == 0 %}
"toolTheHive",
{%- endif %}
- {%- if OSQUERY == 0 %}
+ {%- if not FLEETMANAGER and not FLEETNODE %}
"toolFleet",
{%- endif %}
{%- if GRAFANA == 0 %}
diff --git a/salt/strelka/defaults.yaml b/salt/strelka/defaults.yaml
new file mode 100644
index 000000000..2a3805283
--- /dev/null
+++ b/salt/strelka/defaults.yaml
@@ -0,0 +1,9 @@
+strelka:
+ ignore:
+ - generic_anomalies.yar
+ - general_cloaking.yar
+ - thor_inverse_matches.yar
+ - yara_mixed_ext_vars.yar
+ - gen_susp_js_obfuscatorio.yar
+ - apt_flame2_orchestrator.yar
+ - apt_tetris.yar
diff --git a/salt/strelka/init.sls b/salt/strelka/init.sls
index d90484fed..d66b2bef2 100644
--- a/salt/strelka/init.sls
+++ b/salt/strelka/init.sls
@@ -21,6 +21,8 @@
{% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
{% set STRELKA_RULES = salt['pillar.get']('strelka:rules', '1') %}
{% set ENGINE = salt['pillar.get']('global:mdengine', '') %}
+{% import_yaml 'strelka/defaults.yaml' as strelka_config with context %}
+{% set IGNORELIST = salt['pillar.get']('strelka:ignore', strelka_config.strelka.ignore, merge=True, merge_nested_lists=True) %}
# Strelka config
strelkaconfdir:
@@ -54,6 +56,17 @@ strelkarules:
- source: salt://strelka/rules
- user: 939
- group: 939
+ - clean: True
+ - exclude_pat:
+ {% for IGNOREDRULE in IGNORELIST %}
+ - {{ IGNOREDRULE }}
+ {% endfor %}
+
+ {% for IGNOREDRULE in IGNORELIST %}
+remove_rule_{{ IGNOREDRULE }}:
+ file.absent:
+ - name: /opt/so/conf/strelka/rules/signature-base/{{ IGNOREDRULE }}
+ {% endfor %}
{% if grains['role'] in ['so-eval','so-managersearch', 'so-manager', 'so-standalone', 'so-import'] %}
strelkarepos: