upgrade influx

salt/common/tools/sbin/so-influxdb-clean

@@ -1,45 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-wdurregex="^[0-9]+w$"
-ddurregex="^[0-9]+d$"
-
-echo -e "\nThis script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified."
-echo "The duration will need to be specified as an integer followed by the duration unit without a space."
-echo -e "\nFor example, to purge all data but retain the past 12 weeks, specify 12w for the duration."
-echo "The duration units are as follows:"
-echo "  w  - week(s)"
-echo "  d  - day(s)"
-
-while true; do
-  echo ""
-  read -p 'Enter the duration of past data that you would like to retain: ' duration
-  duration=$(echo $duration | tr '[:upper:]' '[:lower:]')
-
-  if [[ "$duration" =~ $wdurregex ]] || [[ "$duration" =~ $ddurregex ]]; then
-    break
-  fi
-
-  echo -e "\nInvalid duration."
-done
-
-echo -e "\nInfluxDB will now be cleaned and leave only the past $duration worth of data."
-read -r -p "Are you sure you want to continue? [y/N] " yorn
-if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then
-  echo -e "\nCleaning InfluxDb and saving only the past $duration. This may could take several minutes depending on how much data needs to be cleaned."
-    if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - $duration\""; then
-      echo -e "\nInfluxDb clean complete."
-    else
-      echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details."
-    fi
-else
-  echo -e "\nExiting as requested."
-fi

salt/common/tools/sbin/so-influxdb-downsample

@@ -1,55 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-{%- set role = grains.id.split('_') | last %}
-{%- if role in ['manager', 'managersearch', 'eval', 'standalone'] %}
-  {%- import_yaml 'influxdb/defaults.yaml' as default_settings %}
-  {%- set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %}
-
-. /usr/sbin/so-common
-
-echo -e "\nThis script is used to reduce the size of InfluxDB by downsampling old data into the so_long_term retention policy."
-
-echo -e "\nInfluxDB will now be downsampled. This could take a few hours depending on how large the database is and hardware resources available."
-read -r -p "Are you sure you want to continue? [y/N] " yorn
-if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then
-  echo -e "\nDownsampling InfluxDb started at `date`. This may take several hours depending on how much data needs to be downsampled."
-
-  {% for dest_rp in influxdb.downsample.keys() -%}
-    {% for measurement in influxdb.downsample[dest_rp].get('measurements', []) -%}
-
-  day=0
-  startdate=`date`
-  while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".\"{{measurement}}\" FROM \"autogen\".\"{{measurement}}\" WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do
-    # why 2020-07-21? 
-    migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"`
-
-    echo "Downsampling of measurement: {{measurement}} from $migrationdate started at $startdate and completed at `date`."
-
-    newdaytomigrate=$(date -d "$migrationdate + 1 days" +"%s")
-    today=$(date +"%s")
-    if [ $newdaytomigrate -ge $today ]; then
-      break
-    else
-      ((day=day+1))
-      startdate=`date`
-      echo -e "\nDownsampling the next day's worth of data for measurement: {{measurement}}."
-    fi
-  done
-
-    {% endfor -%}
-  {% endfor -%}
-
-  echo -e "\nInfluxDb data downsampling complete."
-
-else
-  echo -e "\nExiting as requested."
-fi
-{%- else %}
-echo -e "\nThis script can only be run on a node running InfluxDB."
-{%- endif %}

salt/common/tools/sbin/so-influxdb-drop-autogen

@@ -1,26 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-
-
-. /usr/sbin/so-common
-
-echo -e "\nThis script is used to reduce the size of InfluxDB by dropping the autogen retention policy."
-echo "If you want to retain historical data prior to 2.3.60, then this should only be run after you have downsampled your data using so-influxdb-downsample."
-
-echo -e "\nThe autogen retention policy will now be dropped from InfluxDB."
-read -r -p "Are you sure you want to continue? [y/N] " yorn
-if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then
-  echo -e "\nDropping autogen retention policy."
-    if docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -execute "drop retention policy autogen on telegraf"; then
-      echo -e "\nAutogen retention policy dropped from InfluxDb."
-    else
-      echo -e "\nSomething went wrong dropping then autogen retention policy from InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details."
-    fi
-else
-  echo -e "\nExiting as requested."
-fi

salt/influxdb/buckets.json.jinja

@@ -1,3 +1 @@
-{%- from 'influxdb/map.jinja' import INFLUXMERGED %}
-
 {{ INFLUXMERGED.buckets | json }}

salt/influxdb/config.yaml.jinja

@@ -1,3 +1 @@
-{%- from 'influxdb/map.jinja' import INFLUXMERGED %}
-
 {{ INFLUXMERGED.config | yaml(false) }}

salt/influxdb/init.sls

@@ -2,6 +2,7 @@
 {% if sls in allowed_states %}
 {% from 'docker/docker.map.jinja' import DOCKER %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
+{% from 'influxdb/map.jinja' import INFLUXMERGED %}
 
 {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-eval', 'so-import'] %}
 {% set PASSWORD = salt['pillar.get']('secrets:influx_pass') %}
@@ -37,6 +38,8 @@ influxdbconf:
     - user: 939
     - group: 939
     - template: jinja
+    - defaults:
+        INFLUXMERGED: {{ INFLUXMERGED }}
 
 influxdbbucketsconf:
   file.managed:
@@ -45,6 +48,8 @@ influxdbbucketsconf:
     - user: 939
     - group: 939
     - template: jinja
+    - defaults:
+        INFLUXMERGED: {{ INFLUXMERGED }}
 
 influxdb-templates:
   file.recurse:
@@ -54,6 +59,8 @@ influxdb-templates:
     - group: 939
     - template: jinja
     - clean: True
+    - defaults:
+        INFLUXMERGED: {{ INFLUXMERGED }}
 
 influxdb_curl_config:
   file.managed:

salt/influxdb/templates/downsample.json

@@ -0,0 +1 @@
+[{"apiVersion":"influxdata.com/v2alpha1","kind":"Task","metadata":{"name":"task-downsample"},"spec":{"every":"{{ INFLUXMERGED.downsample.so_long_term.resolution }}","name":"test","query":"import \"types\"\n\n\n\nfrom(bucket: \"telegraf/so_short_term\")\n    |> range(start: -task.every)\n    |> filter(fn: (r) => types.isNumeric(v: r._value))\n    |> aggregateWindow(every: task.every, fn: mean)\n    |> to(bucket: \"telegraf/so_long_term\")"}}]

salt/salt/files/influxdb_continuous_query.py.patch

@@ -1,4 +0,0 @@
-56c56
-<             database, name, query, resample_time, coverage_period
----
->             database, name, query, resample_time, coverage_period, **client_args

salt/salt/files/influxdb_retention_policy.py.patch

@@ -1,16 +0,0 @@
-34c34
-<         hours = int(duration.split("h"))
----
->         hours = int(duration.split("h")[0])
-48c48
-< def present(name, database, duration="7d", replication=1, default=False, **client_args):
----
-> def present(name, database, duration="7d", replication=1, default=False, shard_duration="1d", **client_args):
-73c73
-<             database, name, duration, replication, default, **client_args
----
->             database, name, duration, replication, shard_duration, default, **client_args
-113c113
-<                     database, name, duration, replication, default, **client_args
----
->                     database, name, duration, replication, shard_duration, default, **client_args

salt/salt/files/influxdbmod.py.patch

@@ -1,16 +0,0 @@
-421c421
-<     database, name, duration, replication, default=False, **client_args
----
->     database, name, duration, replication, shard_duration, default=False, **client_args
-456c456
-<     client.create_retention_policy(name, duration, replication, database, default)
----
->     client.create_retention_policy(name, duration, replication, database, default, shard_duration)
-462c462
-<     database, name, duration, replication, default=False, **client_args
----
->     database, name, duration, replication, shard_duration, default=False, **client_args
-498c498
-<     client.alter_retention_policy(name, database, duration, replication, default)
----
->     client.alter_retention_policy(name, database, duration, replication, default, shard_duration)

salt/salt/map.jinja

@@ -7,10 +7,6 @@
   {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %}
   {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %}
   {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %}
-  {% set PYTHONINFLUXVERSION = '5.3.1' %}
-  {% set PYTHON3INFLUX= 'influxdb == ' ~ PYTHONINFLUXVERSION  %}
-  {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %}
-  {% set PYTHONINSTALLER = 'pip' %}
   {% set SYSTEMD_UNIT_FILE = '/lib/systemd/system/salt-minion.service' %}
 {% else %}
   {% set SPLITCHAR = '-' %}
@@ -18,10 +14,6 @@
   {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion'] %}
   {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %}
   {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %}
-  {% set PYTHONINFLUXVERSION = '5.3.1' %}
-  {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %}
-  {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %}
-  {% set PYTHONINSTALLER = 'pkg' %}
   {% set SYSTEMD_UNIT_FILE = '/usr/lib/systemd/system/salt-minion.service' %}
 {% endif %}
 

salt/salt/minion.sls

@@ -34,21 +34,6 @@ install_salt_minion:
         exec 2>&- # close stderr
         nohup /bin/sh -c '{{ UPGRADECOMMAND }}' &
 
-  {# if we are the salt master #}
-  {% if grains.id.split('_')|first == grains.master %}
-remove_influxdb_continuous_query_state_file:
-  file.absent:
-    - name: /opt/so/state/influxdb_continuous_query.py.patched
-
-remove_influxdbmod_state_file:
-  file.absent:
-    - name: /opt/so/state/influxdbmod.py.patched
-
-remove_influxdb_retention_policy_state_file:
-  file.absent:
-    - name: /opt/so/state/influxdb_retention_policy.py.patched
-  {% endif %}
-
 {% endif %}
 
 {% if INSTALLEDSALTVERSION|string == SALTVERSION|string %}

salt/salt/python3-influxdb.sls

@@ -1,70 +0,0 @@
-{% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %}
-{% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %}
-{% from "salt/map.jinja" import PYTHON3INFLUX with context %}
-{% from "salt/map.jinja" import PYTHON3INFLUXDEPS with context %}
-{% from "salt/map.jinja" import PYTHONINSTALLER with context %}
-
-include:
-  - salt.helper-packages
-
-python3_influxdb_dependencies:
-  {{PYTHONINSTALLER}}.installed:
-    - pkgs: {{ PYTHON3INFLUXDEPS }}
-
-python3_influxdb:
-  {{PYTHONINSTALLER}}.installed:
-    - name: {{ PYTHON3INFLUX }}
-
-# We circumvent the file.patch state putting ERROR in the log by using the unless and file.touch below
-# https://github.com/saltstack/salt/pull/47010 and https://github.com/saltstack/salt/issues/52329
-
-#https://github.com/saltstack/salt/issues/59766
-influxdb_continuous_query.present_patch:
-  file.patch:
-    - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py
-    - source: salt://salt/files/influxdb_continuous_query.py.patch
-    - require:
-      - {{PYTHONINSTALLER}}: python3_influxdb
-      - pkg: patch_package
-    - unless: ls /opt/so/state/influxdb_continuous_query.py.patched
-
-influxdb_continuous_query.py.patched:
-  file.touch:
-    - name: /opt/so/state/influxdb_continuous_query.py.patched
-    - onchanges:
-      - file: influxdb_continuous_query.present_patch
-
-#https://github.com/saltstack/salt/issues/59761
-influxdb_retention_policy.present_patch:
-  file.patch:
-    - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py
-    - source: salt://salt/files/influxdb_retention_policy.py.patch
-    - require:
-      - {{PYTHONINSTALLER}}: python3_influxdb
-      - pkg: patch_package
-    - unless: ls /opt/so/state/influxdb_retention_policy.py.patched
-
-influxdb_retention_policy.py.patched:
-  file.touch:
-    - name: /opt/so/state/influxdb_retention_policy.py.patched
-    - onchanges:
-      - file: influxdb_retention_policy.present_patch
-
-# We should be able to set reload_modules: True in this state in order to tell salt to reload its python modules due to us possibly installing
-# and possibly modifying modules in this state. This is bugged according to https://github.com/saltstack/salt/issues/24925
-influxdbmod.py_shard_duration_patch:
-  file.patch:
-    - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py
-    - source: salt://salt/files/influxdbmod.py.patch
-    - require:
-      - {{PYTHONINSTALLER}}: python3_influxdb
-      - pkg: patch_package
-    - unless: ls /opt/so/state/influxdbmod.py.patched
-
-influxdbmod.py.patched:
-  file.touch:
-    - name: /opt/so/state/influxdbmod.py.patched
-    - onchanges:
-      - file: influxdbmod.py_shard_duration_patch
-
-