CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4664 from Security-Onion-Solutions/influx_scripts

Browse Source 
so-influxdb-downsample script improvements
This commit is contained in:
Josh Patterson
2021-07-01 10:28:21 -04:00
committed by GitHub
parent 26ff50f85c 21c9c7b8f4
commit 4eaef94454
1 changed files with 19 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
salt/common/tools/sbin/so-influxdb-downsample
View File

@@ -14,6 +14,10 @@
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
{%- set role = grains.id.split('_') | last %}
{%- if role in ['manager', 'managersearch', 'eval', 'standalone'] %}
{%- import_yaml 'influxdb/defaults.yaml' as default_settings %}
{%- set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %}
. /usr/sbin/so-common . /usr/sbin/so-common
@@ -22,14 +26,18 @@ echo -e "\nThis script is used to reduce the size of InfluxDB by downsampling ol
echo -e "\nInfluxDB will now be downsampled. This could take a few hours depending on how large the database is and hardware resources available." echo -e "\nInfluxDB will now be downsampled. This could take a few hours depending on how large the database is and hardware resources available."
read -r -p "Are you sure you want to continue? [y/N] " yorn read -r -p "Are you sure you want to continue? [y/N] " yorn
if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then
echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled." echo -e "\nDownsampling InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled."
{% for dest_rp in influxdb.downsample.keys() -%}
{% for measurement in influxdb.downsample[dest_rp].get('measurements', []) -%}
day=0 day=0
startdate=`date` startdate=`date`
while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".\"{{measurement}}\" FROM \"autogen\".\"{{measurement}}\" WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do
while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM \"autogen\"./.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do
# why 2020-07-21? # why 2020-07-21?
migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"` migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"`
echo "Downsampling of $migrationdate started at $startdate and completed at `date`."
echo "Downsampling of measurement: {{measurement}} from $migrationdate started at $startdate and completed at `date`."
newdaytomigrate=$(date -d "$migrationdate + 1 days" +"%s") newdaytomigrate=$(date -d "$migrationdate + 1 days" +"%s")
today=$(date +"%s") today=$(date +"%s")
@@ -38,12 +46,18 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then
else else
((day=day+1)) ((day=day+1))
startdate=`date` startdate=`date`
echo -e "\nDownsampling the next day's worth of data." echo -e "\nDownsampling the next day's worth of data for measurement: {{measurement}}."
fi fi
done done
{% endfor -%}
{% endfor -%}
echo -e "\nInfluxDb data downsampling complete." echo -e "\nInfluxDb data downsampling complete."
else else
echo -e "\nExiting as requested." echo -e "\nExiting as requested."
fi fi
{%- else %}
echo -e "\nThis script can only be run on a node running InfluxDB."
{%- endif %}