From aa47a726565f840289b251d17539c3331e404993 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Jun 2021 07:25:51 -0400 Subject: [PATCH 1/2] source common to require root --- salt/common/tools/sbin/so-influxdb-clean | 2 ++ salt/common/tools/sbin/so-influxdb-migrate | 12 +++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-clean b/salt/common/tools/sbin/so-influxdb-clean index 635ac6a02..7b586f03b 100755 --- a/salt/common/tools/sbin/so-influxdb-clean +++ b/salt/common/tools/sbin/so-influxdb-clean @@ -15,6 +15,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +. /usr/sbin/so-common + wdurregex="^[0-9]+w$" ddurregex="^[0-9]+d$" diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate index 4d65cba95..7ab5378f7 100755 --- a/salt/common/tools/sbin/so-influxdb-migrate +++ b/salt/common/tools/sbin/so-influxdb-migrate @@ -15,19 +15,21 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +. /usr/sbin/so-common + echo -e "\nThis script is used to reduce the size of InfluxDB by downsampling old data into the so_long_term retention policy." -echo -e "\nInfluxDB will now be migrated. This could take a few hours depending on how large the database is and hardware resources available." +echo -e "\nInfluxDB will now be downsampled. This could take a few hours depending on how large the database is and hardware resources available." read -r -p "Are you sure you want to continue? [y/N] " yorn if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then - echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be moved." + echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled." day=0 startdate=`date` while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM \"autogen\"./.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do # why 2020-07-21? migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"` - echo "Migration of $migrationdate started at $startdate and completed at `date`." + echo "Downsampling of $migrationdate started at $startdate and completed at `date`." newdaytomigrate=$(date -d "$migrationdate + 1 days" +"%s") today=$(date +"%s") @@ -36,11 +38,11 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then else ((day=day+1)) startdate=`date` - echo -e "\nMigrating the next day's worth of data." + echo -e "\nDownsampling the next day's worth of data." fi done - echo -e "\nInfluxDb data migration complete." + echo -e "\nInfluxDb data downsampling complete." else echo -e "\nExiting as requested." From ada729087d1ea1743b1b7f7ca7458275da81e44d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Jun 2021 08:14:52 -0400 Subject: [PATCH 2/2] add script to drop autogen, rename so-influxdb-migrate to so-influxdb-downsample --- ...nfluxdb-migrate => so-influxdb-downsample} | 0 .../tools/sbin/so-influxdb-drop-autogen | 34 +++++++++++++++++++ 2 files changed, 34 insertions(+) rename salt/common/tools/sbin/{so-influxdb-migrate => so-influxdb-downsample} (100%) create mode 100644 salt/common/tools/sbin/so-influxdb-drop-autogen diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-downsample similarity index 100% rename from salt/common/tools/sbin/so-influxdb-migrate rename to salt/common/tools/sbin/so-influxdb-downsample diff --git a/salt/common/tools/sbin/so-influxdb-drop-autogen b/salt/common/tools/sbin/so-influxdb-drop-autogen new file mode 100644 index 000000000..56c00234e --- /dev/null +++ b/salt/common/tools/sbin/so-influxdb-drop-autogen @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +. /usr/sbin/so-common + +echo -e "\nThis script is used to reduce the size of InfluxDB by dropping the autogen retention policy." +echo "If you want to retain historical data prior to 2.3.60, then this should only be run after you have downsampled your data using so-influxdb-downsample." + +echo -e "\nThe autogen retention policy will now be dropped from InfluxDB." +read -r -p "Are you sure you want to continue? [y/N] " yorn +if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then + echo -e "\nDropping autogen retention policy." + if docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -execute "drop retention policy autogen on telegraf"; then + echo -e "\nAutogen retention policy dropped from InfluxDb." + else + echo -e "\nSomething went wrong dropping then autogen retention policy from InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." + fi +else + echo -e "\nExiting as requested." +fi