CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2026-01-29 03:13:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

upgrade influx

Browse Source
This commit is contained in:
Jason Ertel
2023-02-13 09:51:48 -05:00
parent 34d19e308f
commit 47af14c265
14 changed files with 9 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
salt/common/tools/sbin/so-influxdb-clean
View File

@@ -1,45 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
wdurregex="^[0-9]+w$"
ddurregex="^[0-9]+d$"
echo -e "\nThis script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified."
echo "The duration will need to be specified as an integer followed by the duration unit without a space."
echo -e "\nFor example, to purge all data but retain the past 12 weeks, specify 12w for the duration."
echo "The duration units are as follows:"
echo " w - week(s)"
echo " d - day(s)"
while true; do
echo ""
read -p 'Enter the duration of past data that you would like to retain: ' duration
duration=$(echo $duration | tr '[:upper:]' '[:lower:]')
if [[ "$duration" =~ $wdurregex ]] || [[ "$duration" =~ $ddurregex ]]; then
break
fi
echo -e "\nInvalid duration."
done
echo -e "\nInfluxDB will now be cleaned and leave only the past $duration worth of data."
read -r -p "Are you sure you want to continue? [y/N] " yorn
if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then
echo -e "\nCleaning InfluxDb and saving only the past $duration. This may could take several minutes depending on how much data needs to be cleaned."
if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - $duration\""; then
echo -e "\nInfluxDb clean complete."
else
echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details."
fi
else
echo -e "\nExiting as requested."
fi

55
salt/common/tools/sbin/so-influxdb-downsample
View File

@@ -1,55 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
{%- set role = grains.id.split('_') | last %}
{%- if role in ['manager', 'managersearch', 'eval', 'standalone'] %}
{%- import_yaml 'influxdb/defaults.yaml' as default_settings %}
{%- set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %}
. /usr/sbin/so-common
echo -e "\nThis script is used to reduce the size of InfluxDB by downsampling old data into the so_long_term retention policy."
echo -e "\nInfluxDB will now be downsampled. This could take a few hours depending on how large the database is and hardware resources available."
read -r -p "Are you sure you want to continue? [y/N] " yorn
if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then
echo -e "\nDownsampling InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled."
{% for dest_rp in influxdb.downsample.keys() -%}
{% for measurement in influxdb.downsample[dest_rp].get('measurements', []) -%}
day=0
startdate=`date`
while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".\"{{measurement}}\" FROM \"autogen\".\"{{measurement}}\" WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do
# why 2020-07-21?
migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"`
echo "Downsampling of measurement: {{measurement}} from $migrationdate started at $startdate and completed at `date`."
newdaytomigrate=$(date -d "$migrationdate + 1 days" +"%s")
today=$(date +"%s")
if [ $newdaytomigrate -ge $today ]; then
break
else
((day=day+1))
startdate=`date`
echo -e "\nDownsampling the next day's worth of data for measurement: {{measurement}}."
fi
done
{% endfor -%}
{% endfor -%}
echo -e "\nInfluxDb data downsampling complete."
else
echo -e "\nExiting as requested."
fi
{%- else %}
echo -e "\nThis script can only be run on a node running InfluxDB."
{%- endif %}

26
salt/common/tools/sbin/so-influxdb-drop-autogen
View File

@@ -1,26 +0,0 @@
#!/bin/bash
# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
# https://securityonion.net/license; you may not use this file except in compliance with the
# Elastic License 2.0.
. /usr/sbin/so-common
echo -e "\nThis script is used to reduce the size of InfluxDB by dropping the autogen retention policy."
echo "If you want to retain historical data prior to 2.3.60, then this should only be run after you have downsampled your data using so-influxdb-downsample."
echo -e "\nThe autogen retention policy will now be dropped from InfluxDB."
read -r -p "Are you sure you want to continue? [y/N] " yorn
if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then
echo -e "\nDropping autogen retention policy."
if docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -execute "drop retention policy autogen on telegraf"; then
echo -e "\nAutogen retention policy dropped from InfluxDb."
else
echo -e "\nSomething went wrong dropping then autogen retention policy from InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details."
fi
else
echo -e "\nExiting as requested."
fi