From 75012cdcbabb66980a551ab2bca4ca2cde709fb5 Mon Sep 17 00:00:00 2001
From: m0duspwnens <josh.patterson@securityonionsolutions.com>
Date: Wed, 10 Mar 2021 15:20:11 -0500
Subject: [PATCH] create rps and cqs

---
 salt/common/init.sls        |  3 ++
 salt/influxdb/defaults.yaml | 33 ++++++++++++++++++
 salt/influxdb/init.sls      | 68 ++++++++++++++++++++++++++-----------
 3 files changed, 85 insertions(+), 19 deletions(-)
 create mode 100644 salt/influxdb/defaults.yaml

diff --git a/salt/common/init.sls b/salt/common/init.sls
index 16cba3c1a..eb7a413e2 100644
--- a/salt/common/init.sls
+++ b/salt/common/init.sls
@@ -105,6 +105,8 @@ commonpkgs:
       - python3-m2crypto
       - python3-mysqldb
       - git
+      - patch
+
 heldpackages:
   pkg.installed:
     - pkgs:
@@ -140,6 +142,7 @@ commonpkgs:
       - lvm2
       - openssl
       - git
+      - patch
 
 heldpackages:
   pkg.installed:
diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml
new file mode 100644
index 000000000..57193e6fa
--- /dev/null
+++ b/salt/influxdb/defaults.yaml
@@ -0,0 +1,33 @@
+influxdb:
+  retention_policies:
+    autogen:
+      default: True
+      duration: 1h
+    so_long_term:
+      default: False
+      duration: 30d
+  downsample:
+    so_long_term:
+      resolution: 30m
+      measurements:
+        - cpu
+        - disk
+        - diskio
+        - docker_container_cpu
+        - docker_container_mem
+        - docker_container_net
+        - elasticsearch_indices
+        - elasticsearch_jvm
+        - esteps
+        - healthcheck
+        - influxsize
+        - mem
+        - net
+        - pcapage
+        - processes
+        - redisqueue
+        - stenodrop
+        - suridrop
+        - system
+        - zeekcaptureloss
+        - zeekdrop
\ No newline at end of file
diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls
index 871b0a16a..b20e6f64b 100644
--- a/salt/influxdb/init.sls
+++ b/salt/influxdb/init.sls
@@ -5,6 +5,8 @@
 {% set MANAGER = salt['grains.get']('master') %}
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
+{% import_yaml 'influxdb/defaults.yaml' as default_settings %}
+{% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %}
 
 {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %}
 
@@ -57,30 +59,31 @@ append_so-influxdb_so-status.conf:
     - name: /opt/so/conf/so-status/so-status.conf
     - text: so-influxdb
 
-set_autogen_retention_policy:
+{% for rp in influxdb.retention_policies.keys() %}
+{{rp}}_retention_policy:
   influxdb_retention_policy.present:
-    - name: autogen
+    - name: {{rp}}
     - database: telegraf
-    - duration: 1h
+    - duration: {{influxdb.retention_policies[rp].duration}}
     - replication: 1
-    - default: True
+    - default: {{influxdb.retention_policies[rp].get('default', 'False')}}
     - ssl: True
-    - unsafeSsl: True
     - require:
       - docker_container: so-influxdb
+{% endfor %}
 
-set_so_long_term_retention_policy:
-  influxdb_retention_policy.present:
-    - name: so_long_term
+{% for dest_rp in influxdb.downsample.keys() %}
+  {% for measurement in influxdb.downsample[dest_rp].measurements %}
+so_downsample_{{measurement}}_cq:
+  influxdb_continuous_query.present:
+    - name: so_downsample_{{measurement}}_cq
     - database: telegraf
-    - duration: 2h
-    - replication: 1
-    - default: False
+    - query: SELECT mean(*) INTO "{{dest_rp}}"."{{measurement}}" FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}})
     - ssl: True
-    - unsafeSsl: True
     - require:
       - docker_container: so-influxdb
-
+  {% endfor %}
+{% endfor %}
 
 {% endif %}
 
@@ -92,9 +95,36 @@ set_so_long_term_retention_policy:
 
 {% endif %}
 
-#influxdb:
-#  retention_policies:
-#    autogen:
-#      duration: 1h
-#    so_long_term:
-#      duration: 2h
\ No newline at end of file
+influxdb:
+  retention_policies:
+    autogen:
+      default: True
+      duration: 1h
+    so_long_term:
+      default: False
+      duration: 2h
+  downsample:
+    so_long_term:
+      resolution: 30m
+      measurements:
+        - cpu
+        - disk
+        - diskio
+        - docker_container_cpu
+        - docker_container_mem
+        - docker_container_net
+        - elasticsearch_indices
+        - elasticsearch_jvm
+        - esteps
+        - healthcheck
+        - influxsize
+        - mem
+        - net
+        - pcapage
+        - processes
+        - redisqueue
+        - stenodrop
+        - suridrop
+        - system
+        - zeekcaptureloss
+        - zeekdrop
\ No newline at end of file