Merge pull request #4140 from Security-Onion-Solutions/issue/3264

https://github.com/Security-Onion-Solutions/securityonion/issues/3264

salt/common/tools/sbin/so-influxdb-clean

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+wdurregex="^[0-9]+w$"
+ddurregex="^[0-9]+d$"
+
+echo -e "\nThis script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified."
+echo "The duration will need to be specified as an integer followed by the duration unit without a space."
+echo -e "\nFor example, to purge all data but retain the past 12 weeks, specify 12w for the duration."
+echo "The duration units are as follows:"
+echo "  w  - week(s)"
+echo "  d  - day(s)"
+
+while true; do
+  echo ""
+  read -p 'Enter the duration of past data that you would like to retain: ' duration
+  duration=$(echo $duration | tr '[:upper:]' '[:lower:]')
+
+  if [[ "$duration" =~ $wdurregex ]] || [[ "$duration" =~ $ddurregex ]]; then
+    break
+  fi
+
+  echo -e "\nInvalid duration."
+done
+
+echo -e "\nInfluxDB will now be cleaned and leave only the past $duration worth of data."
+read -r -p "Are you sure you want to continue? [y/N] " yorn
+if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then
+  echo -e "\nCleaning InfluxDb and saving only the past $duration. This may could take several minutes depending on how much data needs to be cleaned."
+    if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - $duration\""; then
+      echo -e "\nInfluxDb clean complete."
+    else
+      echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details."
+    fi
+else
+  echo -e "\nExiting as requested."
+fi

salt/common/tools/sbin/so-influxdb-migrate

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+echo -e "\nThis script is used to reduce the size of InfluxDB by downsampling old data into the so_long_term retention policy."
+
+echo -e "\nInfluxDB will now be migrated. This could take a few hours depending on how large the database is and hardware resources available."
+read -r -p "Are you sure you want to continue? [y/N] " yorn
+if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then
+  echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be moved."
+  day=0
+  startdate=`date`
+  
+  while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM \"autogen\"./.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do
+    # why 2020-07-21? 
+    migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"`
+    echo "Migration of $migrationdate started at $startdate and completed at `date`."
+
+    newdaytomigrate=$(date -d "$migrationdate + 1 days" +"%s")
+    today=$(date +"%s")
+    if [ $newdaytomigrate -ge $today ]; then
+      break
+    else
+      ((day=day+1))
+      startdate=`date`
+      echo -e "\nMigrating the next day's worth of data."
+    fi
+  done
+
+  echo -e "\nInfluxDb data migration complete."
+
+else
+  echo -e "\nExiting as requested."
+fi

salt/common/tools/sbin/soup

@@ -23,6 +23,7 @@ POSTVERSION=$INSTALLEDVERSION
 INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'})
 BATCHSIZE=5
 SOUP_LOG=/root/soup.log
+INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log
 WHATWOULDYOUSAYYAHDOHERE=soup
 
 add_common() {
@@ -273,6 +274,7 @@ postupgrade_changes() {
     [[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2
     [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
     [[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40
+    [[ "$POSTVERSION" == 2.3.50 ]] && post_2.3.5X_to_2.3.60
 }
 
 post_rc1_to_2.3.21() {
@@ -293,6 +295,10 @@ post_2.3.30_to_2.3.40() {
   POSTVERSION=2.3.40
 }
 
+post_2.3.5X_to_2.3.60() {
+  POSTVERSION=2.3.60
+}
+
 
 rc1_to_rc2() {
 
@@ -795,6 +801,14 @@ else
   echo "Starting Salt Master service."
   systemctl start salt-master
 
+  # Testing that that salt-master is up by checking that is it connected to itself
+  retry 50 10 "salt-call state.show_top -l error" || exit 1
+
+  echo ""
+  echo "Ensuring python modules for Salt are installed and patched."
+  salt-call state.apply salt.python3-influxdb -l info queue=True
+  echo ""
+
   # Only regenerate osquery packages if Fleet is enabled
   FLEET_MANAGER=$(lookup_pillar fleet_manager)
   FLEET_NODE=$(lookup_pillar fleet_node)
@@ -820,6 +834,10 @@ else
   echo ""
   echo "Starting Salt Master service."
   systemctl start salt-master
+
+  # Testing that that salt-master is up by checking that is it connected to itself
+  retry 50 10 "salt-call state.show_top -l error" || exit 1
+
   echo "Running a highstate. This could take several minutes."
   salt-call state.highstate -l info queue=True
   postupgrade_changes

salt/grafana/dashboards/eval/eval.json

@@ -352,7 +352,7 @@
           ],
           "measurement": "zeekcaptureloss",
           "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
           "refId": "A",
           "resultFormat": "time_series",
           "select": [
@@ -2176,7 +2176,7 @@
           ],
           "measurement": "docker_container_mem",
           "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
           "refId": "A",
           "resultFormat": "time_series",
           "select": [

salt/grafana/dashboards/manager/manager.json

@@ -1647,7 +1647,7 @@
           ],
           "measurement": "influxsize",
           "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
           "refId": "A",
           "resultFormat": "time_series",
           "select": [

salt/grafana/dashboards/managersearch/managersearch.json

@@ -1631,7 +1631,7 @@
           ],
           "measurement": "influxsize",
           "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
           "refId": "A",
           "resultFormat": "time_series",
           "select": [

salt/grafana/dashboards/sensor_nodes/sensor.json

@@ -351,7 +351,7 @@
           ],
           "measurement": "zeekcaptureloss",
           "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
           "refId": "A",
           "resultFormat": "time_series",
           "select": [
@@ -2866,7 +2866,7 @@
           ],
           "measurement": "healthcheck",
           "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
           "refId": "A",
           "resultFormat": "time_series",
           "select": [

salt/grafana/dashboards/standalone/standalone.json

@@ -4486,7 +4486,7 @@
           ],
           "measurement": "zeekcaptureloss",
           "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
           "refId": "A",
           "resultFormat": "time_series",
           "select": [
@@ -5107,7 +5107,7 @@
           ],
           "measurement": "influxsize",
           "orderByTime": "ASC",
-          "policy": "autogen",
+          "policy": "default",
           "refId": "A",
           "resultFormat": "time_series",
           "select": [

salt/influxdb/defaults.yaml

@@ -0,0 +1,13 @@
+influxdb:
+  retention_policies:
+    so_short_term:
+      default: True
+      duration: 30d
+      shard_duration: 1d
+    so_long_term:
+      default: False
+      duration: 0d
+      shard_duration: 7d
+  downsample:
+    so_long_term:
+      resolution: 5m

salt/influxdb/init.sls

@@ -5,9 +5,18 @@
 {% set MANAGER = salt['grains.get']('master') %}
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
+{% import_yaml 'influxdb/defaults.yaml' as default_settings %}
+{% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %}
+{% from 'salt/map.jinja' import PYTHON3INFLUX with context %}
+{% from 'salt/map.jinja' import  PYTHONINFLUXVERSION with context %}
+{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c 'import influxdb; print (influxdb.__version__)'", python_shell=True) %}
 
 {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %}
 
+include:
+  - salt.minion
+  - salt.python3-influxdb
+
 # Influx DB
 influxconfdir:
   file.directory:
@@ -57,6 +66,70 @@ append_so-influxdb_so-status.conf:
     - name: /opt/so/conf/so-status/so-status.conf
     - text: so-influxdb
 
+# We have to make sure the influxdb module is the right version prior to state run since reload_modules is bugged
+{% if PYTHONINFLUXVERSIONINSTALLED == PYTHONINFLUXVERSION %}
+wait_for_influxdb:
+  http.query:
+    - name: 'https://{{MANAGER}}:8086/query?q=SHOW+DATABASES'
+    - ssl: True
+    - verify_ssl: False
+    - status: 200
+    - timeout: 30
+    - retry:
+        attempts: 5
+        interval: 60
+
+telegraf_database:
+  influxdb_database.present:
+    - name: telegraf
+    - database: telegraf
+    - ssl: True
+    - verify_ssl: /etc/pki/ca.crt
+    - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key']
+    - influxdb_host: {{ MANAGER }}
+    - require:
+      - docker_container: so-influxdb
+      - sls: salt.python3-influxdb
+      - http: wait_for_influxdb
+
+{% for rp in influxdb.retention_policies.keys() %}
+{{rp}}_retention_policy:
+  influxdb_retention_policy.present:
+    - name: {{rp}}
+    - database: telegraf
+    - duration: {{influxdb.retention_policies[rp].duration}}
+    - shard_duration: {{influxdb.retention_policies[rp].shard_duration}}
+    - replication: 1
+    - default: {{influxdb.retention_policies[rp].get('default', 'False')}}
+    - ssl: True
+    - verify_ssl: /etc/pki/ca.crt
+    - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key']
+    - influxdb_host: {{ MANAGER }}
+    - require:
+      - docker_container: so-influxdb
+      - influxdb_database: telegraf_database
+      - file: influxdb_retention_policy.present_patch
+      - sls: salt.python3-influxdb
+{% endfor %}
+
+{% for dest_rp in influxdb.downsample.keys() %}
+so_downsample_cq:
+  influxdb_continuous_query.present:
+    - name: so_downsample_cq
+    - database: telegraf
+    - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM /.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),*
+    - ssl: True
+    - verify_ssl: /etc/pki/ca.crt
+    - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key']
+    - influxdb_host: {{ MANAGER }}
+    - require:
+      - docker_container: so-influxdb
+      - influxdb_database: telegraf_database
+      - file: influxdb_continuous_query.present_patch
+      - sls: salt.python3-influxdb
+{% endfor %}
+
+{% endif %}
 {% endif %}
 
 {% else %}

salt/salt/files/influxdb_continuous_query.py.patch

@@ -0,0 +1,4 @@
+60c60
+<             database, name, query, resample_time, coverage_period
+---
+>             database, name, query, resample_time, coverage_period, **client_args

salt/salt/files/influxdb_retention_policy.py.patch

@@ -0,0 +1,16 @@
+38c38
+<         hours = int(duration.split("h"))
+---
+>         hours = int(duration.split("h")[0])
+52c52
+< def present(name, database, duration="7d", replication=1, default=False, **client_args):
+---
+> def present(name, database, duration="7d", replication=1, default=False, shard_duration="1d", **client_args):
+77c77
+<             database, name, duration, replication, default, **client_args
+---
+>             database, name, duration, replication, shard_duration, default, **client_args
+119c119
+<                     database, name, duration, replication, default, **client_args
+---
+>                     database, name, duration, replication, shard_duration, default, **client_args

salt/salt/files/influxdbmod.py.patch

@@ -0,0 +1,16 @@
+427c427
+<     database, name, duration, replication, default=False, **client_args
+---
+>     database, name, duration, replication, shard_duration, default=False, **client_args
+462c462
+<     client.create_retention_policy(name, duration, replication, database, default)
+---
+>     client.create_retention_policy(name, duration, replication, database, default, shard_duration)
+468c468
+<     database, name, duration, replication, default=False, **client_args
+---
+>     database, name, duration, replication, shard_duration, default=False, **client_args
+504c504
+<     client.alter_retention_policy(name, database, duration, replication, default)
+---
+>     client.alter_retention_policy(name, database, duration, replication, default, shard_duration)

salt/salt/helper-packages.sls

@@ -0,0 +1,3 @@
+patch_package:
+  pkg.installed:
+    - name: patch

salt/salt/map.jinja

@@ -5,10 +5,22 @@
   {% set SPLITCHAR = '+' %}
   {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep -q salt ; echo $?', python_shell=True) %}
   {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %}
+  {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %}
+  {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %}
+  {% set PYTHONINFLUXVERSION = '5.3.1' %}
+  {% set PYTHON3INFLUX= 'influxdb == ' ~ PYTHONINFLUXVERSION  %}
+  {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %}
+  {% set PYTHONINSTALLER = 'pip' %}
 {% else %}
   {% set SPLITCHAR = '-' %}
   {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %}
   {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion'] %}
+  {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %}
+  {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %}
+  {% set PYTHONINFLUXVERSION = '5.3.1' %}
+  {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %}
+  {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %}
+  {% set PYTHONINSTALLER = 'pkg' %}
 {% endif %}
 
 {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %}

salt/salt/minion.sls

@@ -8,6 +8,7 @@
 
 include:
   - salt
+  - salt.helper-packages
   - systemd.reload
 
 {% if INSTALLEDSALTVERSION|string != SALTVERSION|string %}
@@ -71,3 +72,7 @@ salt_minion_service:
     - name: salt-minion
     - enable: True
     - onlyif: test "{{INSTALLEDSALTVERSION}}" == "{{SALTVERSION}}"
+
+patch_pkg:
+  pkg.installed:
+    - name: patch

salt/salt/python3-influxdb.sls

@@ -0,0 +1,44 @@
+{% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %}
+{% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %}
+{% from "salt/map.jinja" import PYTHON3INFLUX with context %}
+{% from "salt/map.jinja" import PYTHON3INFLUXDEPS with context %}
+{% from "salt/map.jinja" import PYTHONINSTALLER with context %}
+
+include:
+  - salt.helper-packages
+
+python3_influxdb_dependencies:
+  {{PYTHONINSTALLER}}.installed:
+    - pkgs: {{ PYTHON3INFLUXDEPS }}
+
+python3_influxdb:
+  {{PYTHONINSTALLER}}.installed:
+    - name: {{ PYTHON3INFLUX }}
+
+#https://github.com/saltstack/salt/issues/59766
+influxdb_continuous_query.present_patch:
+  file.patch:
+    - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py
+    - source: salt://salt/files/influxdb_continuous_query.py.patch
+    - require:
+      - {{PYTHONINSTALLER}}: python3_influxdb
+      - pkg: patch_package
+
+#https://github.com/saltstack/salt/issues/59761
+influxdb_retention_policy.present_patch:
+  file.patch:
+    - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py
+    - source: salt://salt/files/influxdb_retention_policy.py.patch
+    - require:
+      - {{PYTHONINSTALLER}}: python3_influxdb
+      - pkg: patch_package
+
+# We should be able to set reload_modules: True in this state in order to tell salt to reload its python modules due to us possibly installing
+# and possibly modifying modules in this state. This is bugged according to https://github.com/saltstack/salt/issues/24925
+influxdbmod.py_shard_duration_patch:
+  file.patch:
+    - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py
+    - source: salt://salt/files/influxdbmod.py.patch
+    - require:
+      - {{PYTHONINSTALLER}}: python3_influxdb
+      - pkg: patch_package

salt/ssl/init.sls

@@ -79,6 +79,7 @@ removeesp12dir:
     - signing_policy: influxdb
     - public_key: /etc/pki/influxdb.key
     - CN: {{ manager }}
+    - subjectAltName: DNS:{{ HOSTNAME }}
     - days_remaining: 0
     - days_valid: 820
     - backup: True

setup/so-functions

@@ -2197,9 +2197,9 @@ saltify() {
 		retry 50 10 "apt-get -y install salt-minion=3003+ds-1 salt-common=3003+ds-1" >> "$setup_log" 2>&1 || exit 1
 		retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1
 		if [[ $OSVER != 'xenial' ]]; then
-			retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging" >> "$setup_log" 2>&1 || exit 1
+			retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-packaging python3-influxdb" >> "$setup_log" 2>&1 || exit 1
 		else
-			retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb python-packaging"  >> "$setup_log" 2>&1 || exit 1
+			retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb python-packaging python-influxdb"  >> "$setup_log" 2>&1 || exit 1
 		fi
 	fi
 }

setup/so-setup

@@ -918,10 +918,11 @@ success=$(tail -10 $setup_log | grep Failed | awk '{ print $2}')
 if [[ $success != 0 ]]; then SO_ERROR=1; fi
 
 # Check entire setup log for errors or unexpected salt states and ensure cron jobs are not reporting errors to root's mailbox
-if grep -q -E "ERROR|Result: False" $setup_log || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then 
+# Ignore "Status .* was not found" due to output from salt http.query or http.wait_for_successful_query states used with retry
+if grep -q -E "ERROR|Result: False" $setup_log | grep -qvE "Status .* was not found" || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then 
 	SO_ERROR=1
 
-	grep --color=never "ERROR" "$setup_log" > "$error_log"
+	grep --color=never "ERROR" "$setup_log" | grep -qvE "Status .* was not found" > "$error_log"
 fi
 
 if [[ -n $SO_ERROR ]]; then