From 7b281abf0c733768d5cf20af84bd3603e7456864 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Jun 2021 17:21:18 -0400 Subject: [PATCH 1/5] migrate script now goes through each day and measurement --- salt/common/tools/sbin/so-influxdb-downsample | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-downsample b/salt/common/tools/sbin/so-influxdb-downsample index 7ab5378f7..74a48e6a1 100755 --- a/salt/common/tools/sbin/so-influxdb-downsample +++ b/salt/common/tools/sbin/so-influxdb-downsample @@ -14,6 +14,7 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +{-% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} . /usr/sbin/so-common @@ -25,11 +26,15 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled." day=0 startdate=`date` - - while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM \"autogen\"./.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do + +{% for dest_rp in influxdb.downsample.keys() %} + {% for measurement in influxdb.downsample[dest_rp].get('measurements', []) %} + + while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".\"{{measurement}}\" FROM \"autogen\".\"{{measurement}}\" WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do # why 2020-07-21? migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"` - echo "Downsampling of $migrationdate started at $startdate and completed at `date`." + + echo "Downsampling of measurement: {{measurement}} from $migrationdate started at $startdate and completed at `date`." newdaytomigrate=$(date -d "$migrationdate + 1 days" +"%s") today=$(date +"%s") @@ -38,10 +43,13 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then else ((day=day+1)) startdate=`date` - echo -e "\nDownsampling the next day's worth of data." + echo -e "\nDownsampling the next day's worth of data for measurement: {{measurement}}." fi done + {% endfor %} +{% endfor %} + echo -e "\nInfluxDb data downsampling complete." else From 9235bb35a1dc10b5fe13b6e712f6901022f6020d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Jun 2021 17:30:33 -0400 Subject: [PATCH 2/5] fix jinja whatspace and add defaults --- salt/common/tools/sbin/so-influxdb-downsample | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-influxdb-downsample b/salt/common/tools/sbin/so-influxdb-downsample index 74a48e6a1..b561d2b6a 100755 --- a/salt/common/tools/sbin/so-influxdb-downsample +++ b/salt/common/tools/sbin/so-influxdb-downsample @@ -14,7 +14,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{-% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} +{%- import_yaml 'influxdb/defaults.yaml' as default_settings %} +{%- set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} . /usr/sbin/so-common From eb8a030966f322e019a0a1f429279f6834022dfc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Jun 2021 17:41:38 -0400 Subject: [PATCH 3/5] reset vars in jinja loop --- salt/common/tools/sbin/so-influxdb-downsample | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-downsample b/salt/common/tools/sbin/so-influxdb-downsample index b561d2b6a..ab8fa1dc0 100755 --- a/salt/common/tools/sbin/so-influxdb-downsample +++ b/salt/common/tools/sbin/so-influxdb-downsample @@ -25,12 +25,13 @@ echo -e "\nInfluxDB will now be downsampled. This could take a few hours dependi read -r -p "Are you sure you want to continue? [y/N] " yorn if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled." - day=0 - startdate=`date` + {% for dest_rp in influxdb.downsample.keys() %} {% for measurement in influxdb.downsample[dest_rp].get('measurements', []) %} + day=0 + startdate=`date` while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".\"{{measurement}}\" FROM \"autogen\".\"{{measurement}}\" WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do # why 2020-07-21? migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"` From 108fb1261233c84186b4f2ea1fc23dd7222216f4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 30 Jun 2021 17:53:09 -0400 Subject: [PATCH 4/5] s/Migrating/Downsampling --- salt/common/tools/sbin/so-influxdb-downsample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-influxdb-downsample b/salt/common/tools/sbin/so-influxdb-downsample index ab8fa1dc0..1ecc92611 100755 --- a/salt/common/tools/sbin/so-influxdb-downsample +++ b/salt/common/tools/sbin/so-influxdb-downsample @@ -24,7 +24,7 @@ echo -e "\nThis script is used to reduce the size of InfluxDB by downsampling ol echo -e "\nInfluxDB will now be downsampled. This could take a few hours depending on how large the database is and hardware resources available." read -r -p "Are you sure you want to continue? [y/N] " yorn if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then - echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled." + echo -e "\nDownsampling InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled." {% for dest_rp in influxdb.downsample.keys() %} From 21c9c7b8f45f80956f6fc7f86cebada71f2aed86 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 1 Jul 2021 07:56:45 -0400 Subject: [PATCH 5/5] only render main script if a manager type node --- salt/common/tools/sbin/so-influxdb-downsample | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-downsample b/salt/common/tools/sbin/so-influxdb-downsample index 1ecc92611..d1e32ef9f 100755 --- a/salt/common/tools/sbin/so-influxdb-downsample +++ b/salt/common/tools/sbin/so-influxdb-downsample @@ -14,8 +14,10 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -{%- import_yaml 'influxdb/defaults.yaml' as default_settings %} -{%- set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} +{%- set role = grains.id.split('_') | last %} +{%- if role in ['manager', 'managersearch', 'eval', 'standalone'] %} + {%- import_yaml 'influxdb/defaults.yaml' as default_settings %} + {%- set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} . /usr/sbin/so-common @@ -26,9 +28,8 @@ read -r -p "Are you sure you want to continue? [y/N] " yorn if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then echo -e "\nDownsampling InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled." - -{% for dest_rp in influxdb.downsample.keys() %} - {% for measurement in influxdb.downsample[dest_rp].get('measurements', []) %} + {% for dest_rp in influxdb.downsample.keys() -%} + {% for measurement in influxdb.downsample[dest_rp].get('measurements', []) -%} day=0 startdate=`date` @@ -49,11 +50,14 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then fi done - {% endfor %} -{% endfor %} + {% endfor -%} + {% endfor -%} echo -e "\nInfluxDb data downsampling complete." else echo -e "\nExiting as requested." fi +{%- else %} +echo -e "\nThis script can only be run on a node running InfluxDB." +{%- endif %}