mirror of
https://github.com/Security-Onion-Solutions/securityonion.git
synced 2026-03-06 12:56:24 +01:00
Merge branch 'dev' into issue/3220
This commit is contained in:
William Wernert
51
salt/common/tools/sbin/so-influxdb-clean
Normal file
51
salt/common/tools/sbin/so-influxdb-clean
Normal file
@@ -0,0 +1,51 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
wdurregex="^[0-9]+w$"
|
||||
ddurregex="^[0-9]+d$"
|
||||
|
||||
echo -e "\nThis script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified."
|
||||
echo "The duration will need to be specified as an integer followed by the duration unit without a space."
|
||||
echo -e "\nFor example, to purge all data but retain the past 12 weeks, specify 12w for the duration."
|
||||
echo "The duration units are as follows:"
|
||||
echo " w - week(s)"
|
||||
echo " d - day(s)"
|
||||
|
||||
while true; do
|
||||
echo ""
|
||||
read -p 'Enter the duration of past data that you would like to retain: ' duration
|
||||
duration=$(echo $duration | tr '[:upper:]' '[:lower:]')
|
||||
|
||||
if [[ "$duration" =~ $wdurregex ]] || [[ "$duration" =~ $ddurregex ]]; then
|
||||
break
|
||||
fi
|
||||
|
||||
echo -e "\nInvalid duration."
|
||||
done
|
||||
|
||||
echo -e "\nInfluxDB will now be cleaned and leave only the past $duration worth of data."
|
||||
read -r -p "Are you sure you want to continue? [y/N] " yorn
|
||||
if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then
|
||||
echo -e "\nCleaning InfluxDb and saving only the past $duration. This may could take several minutes depending on how much data needs to be cleaned."
|
||||
if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - $duration\""; then
|
||||
echo -e "\nInfluxDb clean complete."
|
||||
else
|
||||
echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details."
|
||||
fi
|
||||
else
|
||||
echo -e "\nExiting as requested."
|
||||
fi
|
||||
47
salt/common/tools/sbin/so-influxdb-migrate
Normal file
47
salt/common/tools/sbin/so-influxdb-migrate
Normal file
@@ -0,0 +1,47 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
echo -e "\nThis script is used to reduce the size of InfluxDB by downsampling old data into the so_long_term retention policy."
|
||||
|
||||
echo -e "\nInfluxDB will now be migrated. This could take a few hours depending on how large the database is and hardware resources available."
|
||||
read -r -p "Are you sure you want to continue? [y/N] " yorn
|
||||
if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then
|
||||
echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be moved."
|
||||
day=0
|
||||
startdate=`date`
|
||||
|
||||
while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM \"autogen\"./.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do
|
||||
# why 2020-07-21?
|
||||
migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"`
|
||||
echo "Migration of $migrationdate started at $startdate and completed at `date`."
|
||||
|
||||
newdaytomigrate=$(date -d "$migrationdate + 1 days" +"%s")
|
||||
today=$(date +"%s")
|
||||
if [ $newdaytomigrate -ge $today ]; then
|
||||
break
|
||||
else
|
||||
((day=day+1))
|
||||
startdate=`date`
|
||||
echo -e "\nMigrating the next day's worth of data."
|
||||
fi
|
||||
done
|
||||
|
||||
echo -e "\nInfluxDb data migration complete."
|
||||
|
||||
else
|
||||
echo -e "\nExiting as requested."
|
||||
fi
|
||||
@@ -23,6 +23,7 @@ POSTVERSION=$INSTALLEDVERSION
|
||||
INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk '{print $2}')
|
||||
BATCHSIZE=5
|
||||
SOUP_LOG=/root/soup.log
|
||||
INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log
|
||||
WHATWOULDYOUSAYYAHDOHERE=soup
|
||||
|
||||
set -e
|
||||
@@ -349,6 +350,7 @@ postupgrade_changes() {
|
||||
[[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2
|
||||
[[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
|
||||
[[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40
|
||||
[[ "$POSTVERSION" == 2.3.50 ]] && post_2.3.5X_to_2.3.60
|
||||
}
|
||||
|
||||
post_rc1_to_2.3.21() {
|
||||
@@ -369,6 +371,10 @@ post_2.3.30_to_2.3.40() {
|
||||
POSTVERSION=2.3.40
|
||||
}
|
||||
|
||||
post_2.3.5X_to_2.3.60() {
|
||||
POSTVERSION=2.3.60
|
||||
}
|
||||
|
||||
|
||||
rc1_to_rc2() {
|
||||
|
||||
@@ -882,6 +888,14 @@ else
|
||||
echo "Starting Salt Master service."
|
||||
systemctl start salt-master
|
||||
|
||||
# Testing that that salt-master is up by checking that is it connected to itself
|
||||
retry 50 10 "salt-call state.show_top -l error" || exit 1
|
||||
|
||||
echo ""
|
||||
echo "Ensuring python modules for Salt are installed and patched."
|
||||
salt-call state.apply salt.python3-influxdb -l info queue=True
|
||||
echo ""
|
||||
|
||||
# Only regenerate osquery packages if Fleet is enabled
|
||||
FLEET_MANAGER=$(lookup_pillar fleet_manager)
|
||||
FLEET_NODE=$(lookup_pillar fleet_node)
|
||||
@@ -907,6 +921,10 @@ else
|
||||
echo ""
|
||||
echo "Starting Salt Master service."
|
||||
systemctl start salt-master
|
||||
|
||||
# Testing that that salt-master is up by checking that is it connected to itself
|
||||
retry 50 10 "salt-call state.show_top -l error" || exit 1
|
||||
|
||||
echo "Running a highstate. This could take several minutes."
|
||||
salt-call state.highstate -l info queue=True
|
||||
postupgrade_changes
|
||||
|
||||
Reference in New Issue
Block a user