From 7409f1575235719bb40de14b6a695fe481a96a4b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 5 Mar 2021 13:59:29 -0500 Subject: [PATCH 01/76] update all grafana queries that were using autogen to use default --- salt/grafana/dashboards/eval/eval.json | 4 ++-- salt/grafana/dashboards/manager/manager.json | 2 +- salt/grafana/dashboards/managersearch/managersearch.json | 2 +- salt/grafana/dashboards/sensor_nodes/sensor.json | 4 ++-- salt/grafana/dashboards/standalone/standalone.json | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/grafana/dashboards/eval/eval.json b/salt/grafana/dashboards/eval/eval.json index cc4298bb2..1b67267a5 100644 --- a/salt/grafana/dashboards/eval/eval.json +++ b/salt/grafana/dashboards/eval/eval.json @@ -352,7 +352,7 @@ ], "measurement": "zeekcaptureloss", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ @@ -2176,7 +2176,7 @@ ], "measurement": "docker_container_mem", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ diff --git a/salt/grafana/dashboards/manager/manager.json b/salt/grafana/dashboards/manager/manager.json index 2ce913155..a53438dd6 100644 --- a/salt/grafana/dashboards/manager/manager.json +++ b/salt/grafana/dashboards/manager/manager.json @@ -1647,7 +1647,7 @@ ], "measurement": "influxsize", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ diff --git a/salt/grafana/dashboards/managersearch/managersearch.json b/salt/grafana/dashboards/managersearch/managersearch.json index 15bf3cc73..1b71a9e84 100644 --- a/salt/grafana/dashboards/managersearch/managersearch.json +++ b/salt/grafana/dashboards/managersearch/managersearch.json @@ -1631,7 +1631,7 @@ ], "measurement": "influxsize", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ diff --git a/salt/grafana/dashboards/sensor_nodes/sensor.json b/salt/grafana/dashboards/sensor_nodes/sensor.json index ea0a6a63b..f9dbc9f9c 100644 --- a/salt/grafana/dashboards/sensor_nodes/sensor.json +++ b/salt/grafana/dashboards/sensor_nodes/sensor.json @@ -351,7 +351,7 @@ ], "measurement": "zeekcaptureloss", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ @@ -2866,7 +2866,7 @@ ], "measurement": "healthcheck", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ diff --git a/salt/grafana/dashboards/standalone/standalone.json b/salt/grafana/dashboards/standalone/standalone.json index 60a5c6c6c..d745f13c8 100644 --- a/salt/grafana/dashboards/standalone/standalone.json +++ b/salt/grafana/dashboards/standalone/standalone.json @@ -4486,7 +4486,7 @@ ], "measurement": "zeekcaptureloss", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ @@ -5107,7 +5107,7 @@ ], "measurement": "influxsize", "orderByTime": "ASC", - "policy": "autogen", + "policy": "default", "refId": "A", "resultFormat": "time_series", "select": [ From 1f9e5ca3ccf1818dee7c6203e408f4c00ddf39d8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 9 Mar 2021 10:31:59 -0500 Subject: [PATCH 02/76] install influxdb python module add test retention policies --- salt/influxdb/init.sls | 32 ++++++++++++++++++++++++++++++++ setup/so-functions | 3 ++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index aace4e827..871b0a16a 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -57,6 +57,31 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb +set_autogen_retention_policy: + influxdb_retention_policy.present: + - name: autogen + - database: telegraf + - duration: 1h + - replication: 1 + - default: True + - ssl: True + - unsafeSsl: True + - require: + - docker_container: so-influxdb + +set_so_long_term_retention_policy: + influxdb_retention_policy.present: + - name: so_long_term + - database: telegraf + - duration: 2h + - replication: 1 + - default: False + - ssl: True + - unsafeSsl: True + - require: + - docker_container: so-influxdb + + {% endif %} {% else %} @@ -66,3 +91,10 @@ append_so-influxdb_so-status.conf: - name: {{sls}}_state_not_allowed {% endif %} + +#influxdb: +# retention_policies: +# autogen: +# duration: 1h +# so_long_term: +# duration: 2h \ No newline at end of file diff --git a/setup/so-functions b/setup/so-functions index 21602f320..5890f4726 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -2022,6 +2022,7 @@ saltify() { set_progress_str 7 'Installing salt-master' retry 50 10 "apt-get -y install salt-master=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-master" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "pip3 install --user influxdb" >> "$setup_log" 2>&1 || exit 1 ;; *) # Copy down the gpg keys and install them from the manager @@ -2041,7 +2042,7 @@ saltify() { retry 50 10 "apt-get -y install salt-minion=3002.5+ds-1 salt-common=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-minion salt-common" >> "$setup_log" 2>&1 || exit 1 if [[ $OSVER != 'xenial' ]]; then - retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-influxdb" >> "$setup_log" 2>&1 || exit 1 else retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb" >> "$setup_log" 2>&1 || exit 1 fi From c1e4c4cb3049382ef8c695a78d019a623f363612 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 9 Mar 2021 11:50:27 -0500 Subject: [PATCH 03/76] fix pip and python-influxdb install --- setup/so-functions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 391ac3558..9ea165548 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1917,7 +1917,8 @@ saltify() { reserve_group_ids >> "$setup_log" 2>&1 if [[ ! $is_iso ]]; then logCmd "yum -y install epel-release" - logCmd "yum -y install sqlite argon2 curl mariadb-devel" + logCmd "yum -y install sqlite argon2 curl mariadb-devel python3-pip" + retry 50 10 "pip3 install --user influxdb" >> "$setup_log" 2>&1 || exit 1 fi # Download Ubuntu Keys in case manager updates = 1 mkdir -p /opt/so/gpg >> "$setup_log" 2>&1 @@ -2032,7 +2033,6 @@ saltify() { set_progress_str 7 'Installing salt-master' retry 50 10 "apt-get -y install salt-master=3002.5+ds-1" >> "$setup_log" 2>&1 || exit 1 retry 50 10 "apt-mark hold salt-master" >> "$setup_log" 2>&1 || exit 1 - retry 50 10 "pip3 install --user influxdb" >> "$setup_log" 2>&1 || exit 1 ;; *) # Copy down the gpg keys and install them from the manager @@ -2054,7 +2054,7 @@ saltify() { if [[ $OSVER != 'xenial' ]]; then retry 50 10 "apt-get -y install python3-pip python3-dateutil python3-m2crypto python3-mysqldb python3-influxdb" >> "$setup_log" 2>&1 || exit 1 else - retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb" >> "$setup_log" 2>&1 || exit 1 + retry 50 10 "apt-get -y install python-pip python-dateutil python-m2crypto python-mysqldb python-influxdb" >> "$setup_log" 2>&1 || exit 1 fi fi } From 75012cdcbabb66980a551ab2bca4ca2cde709fb5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 10 Mar 2021 15:20:11 -0500 Subject: [PATCH 04/76] create rps and cqs --- salt/common/init.sls | 3 ++ salt/influxdb/defaults.yaml | 33 ++++++++++++++++++ salt/influxdb/init.sls | 68 ++++++++++++++++++++++++++----------- 3 files changed, 85 insertions(+), 19 deletions(-) create mode 100644 salt/influxdb/defaults.yaml diff --git a/salt/common/init.sls b/salt/common/init.sls index 16cba3c1a..eb7a413e2 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -105,6 +105,8 @@ commonpkgs: - python3-m2crypto - python3-mysqldb - git + - patch + heldpackages: pkg.installed: - pkgs: @@ -140,6 +142,7 @@ commonpkgs: - lvm2 - openssl - git + - patch heldpackages: pkg.installed: diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml new file mode 100644 index 000000000..57193e6fa --- /dev/null +++ b/salt/influxdb/defaults.yaml @@ -0,0 +1,33 @@ +influxdb: + retention_policies: + autogen: + default: True + duration: 1h + so_long_term: + default: False + duration: 30d + downsample: + so_long_term: + resolution: 30m + measurements: + - cpu + - disk + - diskio + - docker_container_cpu + - docker_container_mem + - docker_container_net + - elasticsearch_indices + - elasticsearch_jvm + - esteps + - healthcheck + - influxsize + - mem + - net + - pcapage + - processes + - redisqueue + - stenodrop + - suridrop + - system + - zeekcaptureloss + - zeekdrop \ No newline at end of file diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 871b0a16a..b20e6f64b 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -5,6 +5,8 @@ {% set MANAGER = salt['grains.get']('master') %} {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %} {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} +{% import_yaml 'influxdb/defaults.yaml' as default_settings %} +{% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} @@ -57,30 +59,31 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb -set_autogen_retention_policy: +{% for rp in influxdb.retention_policies.keys() %} +{{rp}}_retention_policy: influxdb_retention_policy.present: - - name: autogen + - name: {{rp}} - database: telegraf - - duration: 1h + - duration: {{influxdb.retention_policies[rp].duration}} - replication: 1 - - default: True + - default: {{influxdb.retention_policies[rp].get('default', 'False')}} - ssl: True - - unsafeSsl: True - require: - docker_container: so-influxdb +{% endfor %} -set_so_long_term_retention_policy: - influxdb_retention_policy.present: - - name: so_long_term +{% for dest_rp in influxdb.downsample.keys() %} + {% for measurement in influxdb.downsample[dest_rp].measurements %} +so_downsample_{{measurement}}_cq: + influxdb_continuous_query.present: + - name: so_downsample_{{measurement}}_cq - database: telegraf - - duration: 2h - - replication: 1 - - default: False + - query: SELECT mean(*) INTO "{{dest_rp}}"."{{measurement}}" FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}) - ssl: True - - unsafeSsl: True - require: - docker_container: so-influxdb - + {% endfor %} +{% endfor %} {% endif %} @@ -92,9 +95,36 @@ set_so_long_term_retention_policy: {% endif %} -#influxdb: -# retention_policies: -# autogen: -# duration: 1h -# so_long_term: -# duration: 2h \ No newline at end of file +influxdb: + retention_policies: + autogen: + default: True + duration: 1h + so_long_term: + default: False + duration: 2h + downsample: + so_long_term: + resolution: 30m + measurements: + - cpu + - disk + - diskio + - docker_container_cpu + - docker_container_mem + - docker_container_net + - elasticsearch_indices + - elasticsearch_jvm + - esteps + - healthcheck + - influxsize + - mem + - net + - pcapage + - processes + - redisqueue + - stenodrop + - suridrop + - system + - zeekcaptureloss + - zeekdrop \ No newline at end of file From 8fc1656939468b69d6fb2dd5599d1762ed920ae6 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 09:24:57 -0500 Subject: [PATCH 05/76] fix timeouts / retries in ssl state --- salt/ssl/init.sls | 110 +++++++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 44 deletions(-) diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 8d6c65bea..4646b1d07 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -68,8 +68,9 @@ removeesp12dir: - x509: /etc/pki/influxdb.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the talking to influxdb /etc/pki/influxdb.crt: @@ -86,8 +87,9 @@ removeesp12dir: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/influxdb.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 influxkeyperms: file.managed: @@ -111,8 +113,9 @@ influxkeyperms: - x509: /etc/pki/redis.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/redis.crt: x509.certificate_managed: @@ -153,8 +156,9 @@ rediskeyperms: - x509: /etc/pki/filebeat.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Request a cert and drop it where it needs to go to be distributed /etc/pki/filebeat.crt: @@ -175,8 +179,9 @@ rediskeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/filebeat.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 cmd.run: - name: "/usr/bin/openssl pkcs8 -in /etc/pki/filebeat.key -topk8 -out /etc/pki/filebeat.p8 -nocrypt" - onchanges: @@ -232,8 +237,9 @@ fbcrtlink: - x509: /etc/pki/registry.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the docker registry /etc/pki/registry.crt: @@ -250,8 +256,9 @@ fbcrtlink: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/registry.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 regkeyperms: file.managed: @@ -273,8 +280,9 @@ regkeyperms: - x509: /etc/pki/minio.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for minio /etc/pki/minio.crt: @@ -291,8 +299,9 @@ regkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/minio.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 miniokeyperms: file.managed: @@ -332,8 +341,9 @@ miniokeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 cmd.run: - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:" - onchanges: @@ -366,8 +376,9 @@ elasticp12perms: - x509: /etc/pki/managerssl.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the reverse proxy /etc/pki/managerssl.crt: @@ -385,8 +396,9 @@ elasticp12perms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/managerssl.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 msslkeyperms: file.managed: @@ -409,8 +421,9 @@ msslkeyperms: - x509: /etc/pki/fleet.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/fleet.crt: x509.certificate_managed: @@ -425,8 +438,9 @@ msslkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/fleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 fleetkeyperms: file.managed: @@ -456,8 +470,9 @@ fbcertdir: - x509: /opt/so/conf/filebeat/etc/pki/filebeat.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Request a cert and drop it where it needs to go to be distributed /opt/so/conf/filebeat/etc/pki/filebeat.crt: @@ -478,8 +493,9 @@ fbcertdir: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /opt/so/conf/filebeat/etc/pki/filebeat.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Convert the key to pkcs#8 so logstash will work correctly. filebeatpkcs: @@ -520,8 +536,9 @@ chownfilebeatp8: - x509: /etc/pki/managerssl.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 # Create a cert for the reverse proxy /etc/pki/managerssl.crt: @@ -539,8 +556,9 @@ chownfilebeatp8: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/managerssl.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 msslkeyperms: file.managed: @@ -563,8 +581,9 @@ msslkeyperms: - x509: /etc/pki/fleet.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/fleet.crt: x509.certificate_managed: @@ -579,8 +598,9 @@ msslkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/fleet.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 fleetkeyperms: file.managed: @@ -606,8 +626,9 @@ fleetkeyperms: - x509: /etc/pki/elasticsearch.crt {%- endif %} - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 /etc/pki/elasticsearch.crt: x509.certificate_managed: @@ -623,8 +644,9 @@ fleetkeyperms: # Will trigger 5 days (432000 sec) from cert expiration - 'enddate=$(date -d "$(openssl x509 -in /etc/pki/elasticsearch.crt -enddate -noout | cut -d= -f2)" +%s) ; now=$(date +%s) ; expire_date=$(( now + 432000)); [ $enddate -gt $expire_date ]' - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 cmd.run: - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/elasticsearch.key -in /etc/pki/elasticsearch.crt -export -out /etc/pki/elasticsearch.p12 -nodes -passout pass:" - onchanges: From 6dba2879c52e65828598f609e0f409aa35c49300 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 09:25:44 -0500 Subject: [PATCH 06/76] change so_long_term rp to 6h for tetsing --- salt/influxdb/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 57193e6fa..9647b70ea 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -5,7 +5,7 @@ influxdb: duration: 1h so_long_term: default: False - duration: 30d + duration: 6h downsample: so_long_term: resolution: 30m From 3385ba2ca2da7590eae56f513e241adc3fe7360c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 09:35:54 -0500 Subject: [PATCH 07/76] verify ssl --- salt/influxdb/init.sls | 42 +++++++----------------------------------- 1 file changed, 7 insertions(+), 35 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index b20e6f64b..f55e255b7 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -68,6 +68,9 @@ append_so-influxdb_so-status.conf: - replication: 1 - default: {{influxdb.retention_policies[rp].get('default', 'False')}} - ssl: True + - verify_ssl: /etc/pki/ca.crt + - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] + - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb {% endfor %} @@ -80,6 +83,9 @@ so_downsample_{{measurement}}_cq: - database: telegraf - query: SELECT mean(*) INTO "{{dest_rp}}"."{{measurement}}" FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}) - ssl: True + - verify_ssl: /etc/pki/ca.crt + - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] + - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb {% endfor %} @@ -93,38 +99,4 @@ so_downsample_{{measurement}}_cq: test.fail_without_changes: - name: {{sls}}_state_not_allowed -{% endif %} - -influxdb: - retention_policies: - autogen: - default: True - duration: 1h - so_long_term: - default: False - duration: 2h - downsample: - so_long_term: - resolution: 30m - measurements: - - cpu - - disk - - diskio - - docker_container_cpu - - docker_container_mem - - docker_container_net - - elasticsearch_indices - - elasticsearch_jvm - - esteps - - healthcheck - - influxsize - - mem - - net - - pcapage - - processes - - redisqueue - - stenodrop - - suridrop - - system - - zeekcaptureloss - - zeekdrop \ No newline at end of file +{% endif %} \ No newline at end of file From 3b74d987c1a1486dc5e720b93814d592ebc35c62 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 10:49:15 -0500 Subject: [PATCH 08/76] fix retry in ca state. add subjectAltName to influxdb.crt --- salt/ca/init.sls | 5 +++-- salt/ssl/init.sls | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/ca/init.sls b/salt/ca/init.sls index 0d35c10c1..485b01d3e 100644 --- a/salt/ca/init.sls +++ b/salt/ca/init.sls @@ -43,8 +43,9 @@ pki_private_key: - require: - file: /etc/pki - timeout: 30 - - retry: 5 - - interval: 30 + - retry: + attempts: 5 + interval: 30 x509_pem_entries: module.run: diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index 4646b1d07..85854bab4 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -79,6 +79,7 @@ removeesp12dir: - signing_policy: influxdb - public_key: /etc/pki/influxdb.key - CN: {{ manager }} + - subjectAltName: DNS:{{ HOSTNAME }} - days_remaining: 0 - days_valid: 820 - backup: True From c4da57603087b0c6ab9f7bbad5147c439aabc471 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 11 Mar 2021 12:20:32 -0500 Subject: [PATCH 09/76] ensure the presence of the telegraf database --- salt/influxdb/init.sls | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index f55e255b7..608561857 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -59,6 +59,17 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb +telegraf_database: + influxdb_database.present: + - name: telegraf + - database: telegraf + - ssl: True + - verify_ssl: /etc/pki/ca.crt + - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] + - influxdb_host: {{ MANAGER }} + - require: + - docker_container: so-influxdb + {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: influxdb_retention_policy.present: @@ -73,6 +84,7 @@ append_so-influxdb_so-status.conf: - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb + - influxdb_database: telegraf_database {% endfor %} {% for dest_rp in influxdb.downsample.keys() %} @@ -88,6 +100,7 @@ so_downsample_{{measurement}}_cq: - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb + - influxdb_database: telegraf_database {% endfor %} {% endfor %} From 3a8aea0de69ceb26ec4e115c00dd465001a16aae Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Mar 2021 09:11:48 -0400 Subject: [PATCH 10/76] removing domainstats and freqserver from so-image-common --- salt/common/tools/sbin/so-image-common | 2 -- 1 file changed, 2 deletions(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 402ae97f3..a3b9f62e9 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -47,13 +47,11 @@ container_list() { TRUSTED_CONTAINERS=( "so-acng" "so-curator" - "so-domainstats" "so-elastalert" "so-elasticsearch" "so-filebeat" "so-fleet" "so-fleet-launcher" - "so-freqserver" "so-grafana" "so-idstools" "so-influxdb" From 53528d486c0a2793cf45ccda7ee2950bb2f29e4e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Mar 2021 09:44:56 -0400 Subject: [PATCH 11/76] remove minio --- salt/common/tools/sbin/so-image-common | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index a3b9f62e9..be5a327f0 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -58,7 +58,6 @@ container_list() { "so-kibana" "so-kratos" "so-logstash" - "so-minio" "so-mysql" "so-nginx" "so-pcaptools" From 11c3f14b42d64b5581304e383f81b96b6f9b6836 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Mar 2021 10:35:20 -0400 Subject: [PATCH 12/76] end patch files with newline --- salt/salt/files/influxdb_continuous_query.py.patch | 2 +- salt/salt/files/influxdb_retention_policy.py.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/files/influxdb_continuous_query.py.patch b/salt/salt/files/influxdb_continuous_query.py.patch index 049b04782..a596983ff 100644 --- a/salt/salt/files/influxdb_continuous_query.py.patch +++ b/salt/salt/files/influxdb_continuous_query.py.patch @@ -1,4 +1,4 @@ 60c60 < database, name, query, resample_time, coverage_period --- -> database, name, query, resample_time, coverage_period, **client_args \ No newline at end of file +> database, name, query, resample_time, coverage_period, **client_args diff --git a/salt/salt/files/influxdb_retention_policy.py.patch b/salt/salt/files/influxdb_retention_policy.py.patch index b16de431e..79ca3e536 100644 --- a/salt/salt/files/influxdb_retention_policy.py.patch +++ b/salt/salt/files/influxdb_retention_policy.py.patch @@ -1,4 +1,4 @@ 38c38 < hours = int(duration.split("h")) --- -> hours = int(duration.split("h")[0]) \ No newline at end of file +> hours = int(duration.split("h")[0]) From d75fafb19cddfadeeff95be574b6d492b42d145b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 24 Mar 2021 17:30:27 -0400 Subject: [PATCH 13/76] add support for shard_duration to influxdb module and influxdb_retention_policy state - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- .../files/influxdb_retention_policy.py.patch | 12 ++++++++++++ salt/salt/files/influxmod.py.patch | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 salt/salt/files/influxmod.py.patch diff --git a/salt/salt/files/influxdb_retention_policy.py.patch b/salt/salt/files/influxdb_retention_policy.py.patch index 79ca3e536..7180f8934 100644 --- a/salt/salt/files/influxdb_retention_policy.py.patch +++ b/salt/salt/files/influxdb_retention_policy.py.patch @@ -2,3 +2,15 @@ < hours = int(duration.split("h")) --- > hours = int(duration.split("h")[0]) +52c52 +< def present(name, database, duration="7d", replication=1, default=False, **client_args): +--- +> def present(name, database, duration="7d", replication=1, default=False, shard_duration="1d", **client_args): +77c77 +< database, name, duration, replication, default, **client_args +--- +> database, name, duration, replication, shard_duration, default, **client_args +119c119 +< database, name, duration, replication, default, **client_args +--- +> database, name, duration, replication, shard_duration, default, **client_args diff --git a/salt/salt/files/influxmod.py.patch b/salt/salt/files/influxmod.py.patch new file mode 100644 index 000000000..b653265cb --- /dev/null +++ b/salt/salt/files/influxmod.py.patch @@ -0,0 +1,16 @@ +427c427 +< database, name, duration, replication, default=False, **client_args +--- +> database, name, duration, replication, shard_duration, default=False, **client_args +462c462 +< client.create_retention_policy(name, duration, replication, database, default) +--- +> client.create_retention_policy(name, duration, replication, database, default, shard_duration) +468c468 +< database, name, duration, replication, default=False, **client_args +--- +> database, name, duration, replication, shard_duration, default=False, **client_args +504c504 +< client.alter_retention_policy(name, database, duration, replication, default) +--- +> client.alter_retention_policy(name, database, duration, replication, default, shard_duration) From cd0ab5c70909b3234d59a04a5c347aacbf51ec06 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 25 Mar 2021 08:50:32 -0400 Subject: [PATCH 14/76] add support for shard_duration to influxdb module and influxdb_retention_policy state - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- .../files/{influxmod.py.patch => influxdbmod.py.patch} | 0 salt/salt/minion.sls | 7 ++++++- 2 files changed, 6 insertions(+), 1 deletion(-) rename salt/salt/files/{influxmod.py.patch => influxdbmod.py.patch} (100%) diff --git a/salt/salt/files/influxmod.py.patch b/salt/salt/files/influxdbmod.py.patch similarity index 100% rename from salt/salt/files/influxmod.py.patch rename to salt/salt/files/influxdbmod.py.patch diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index fa5c0659e..47cd179ac 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -53,4 +53,9 @@ influxdb_continuous_query.present_patch: influxdb_retention_policy.present_patch: file.patch: - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_retention_policy.py - - source: salt://salt/files/influxdb_retention_policy.py.patch \ No newline at end of file + - source: salt://salt/files/influxdb_retention_policy.py.patch + +influxdbmod.py_shard_duration_patch: + file.patch: + - name: /usr/lib/python3.6/site-packages/salt/modules/influxdbmod.py + - source: salt://salt/files/influxdbmod.py.patch \ No newline at end of file From 889e624a8cbf3736d2f45b57b2a92cb7934dac73 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 25 Mar 2021 09:33:10 -0400 Subject: [PATCH 15/76] add shard_duration to state and defaults - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- salt/influxdb/defaults.yaml | 6 ++++-- salt/influxdb/init.sls | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 9647b70ea..88faaedba 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,10 +2,12 @@ influxdb: retention_policies: autogen: default: True - duration: 1h + duration: 1d + shard_duration: 1h so_long_term: default: False - duration: 6h + duration: 7d + shard_duration: 1d downsample: so_long_term: resolution: 30m diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 88f03042e..59673c93b 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -79,6 +79,7 @@ telegraf_database: - name: {{rp}} - database: telegraf - duration: {{influxdb.retention_policies[rp].duration}} + - shard_duration: {{influxdb.retention_policies[rp].shard_duration}} - replication: 1 - default: {{influxdb.retention_policies[rp].get('default', 'False')}} - ssl: True From e0f4abaa093bb9617ed77b64eeb6eab219bd6b98 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Mar 2021 10:36:56 -0400 Subject: [PATCH 16/76] try to do it with just 1 cq, modify defaults for testing --- salt/influxdb/defaults.yaml | 4 ++-- salt/influxdb/init.sls | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 88faaedba..3de7edd56 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,11 +2,11 @@ influxdb: retention_policies: autogen: default: True - duration: 1d + duration: 1h shard_duration: 1h so_long_term: default: False - duration: 7d + duration: 2d shard_duration: 1d downsample: so_long_term: diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 59673c93b..b3f3e5c9c 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -92,13 +92,14 @@ telegraf_database: - file: influxdb_retention_policy.present_patch {% endfor %} +{# {% for dest_rp in influxdb.downsample.keys() %} {% for measurement in influxdb.downsample[dest_rp].measurements %} so_downsample_{{measurement}}_cq: influxdb_continuous_query.present: - name: so_downsample_{{measurement}}_cq - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}"."{{measurement}}" FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}) + - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}), * - ssl: True - verify_ssl: /etc/pki/ca.crt - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] @@ -109,6 +110,22 @@ so_downsample_{{measurement}}_cq: - file: influxdb_continuous_query.present_patch {% endfor %} {% endfor %} +#} + +so_downsample_cq: + influxdb_continuous_query.present: + - name: so_downsample_cq + - database: telegraf + - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM /.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* + - ssl: True + - verify_ssl: /etc/pki/ca.crt + - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] + - influxdb_host: {{ MANAGER }} + - require: + - docker_container: so-influxdb + - influxdb_database: telegraf_database + - file: influxdb_continuous_query.present_patch + {% endif %} From d1150f150fb174f6aaec9e13c5916ada0ae76e81 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 29 Mar 2021 10:59:18 -0400 Subject: [PATCH 17/76] loop through the rps --- salt/influxdb/init.sls | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index b3f3e5c9c..1b21612b3 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -112,6 +112,7 @@ so_downsample_{{measurement}}_cq: {% endfor %} #} +{% for dest_rp in influxdb.downsample.keys() %} so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq @@ -125,7 +126,7 @@ so_downsample_cq: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - +{% endfor %} {% endif %} From 30c6d4756a76687b9cbf95d9c9ff5085c538bb66 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Mar 2021 09:38:37 -0400 Subject: [PATCH 18/76] change default long term resolution to 5m --- salt/influxdb/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 3de7edd56..927cd7b52 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -10,7 +10,7 @@ influxdb: shard_duration: 1d downsample: so_long_term: - resolution: 30m + resolution: 5m measurements: - cpu - disk From 8e55e0b9944e6d46032b8fa2e00998a3efd9ded3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 30 Mar 2021 13:36:52 -0400 Subject: [PATCH 19/76] start graphing data from so_long_term --- .../dashboards/standalone/standalone.new.json | 7841 +++++++++++++++++ 1 file changed, 7841 insertions(+) create mode 100644 salt/grafana/dashboards/standalone/standalone.new.json diff --git a/salt/grafana/dashboards/standalone/standalone.new.json b/salt/grafana/dashboards/standalone/standalone.new.json new file mode 100644 index 000000000..41c4d3541 --- /dev/null +++ b/salt/grafana/dashboards/standalone/standalone.new.json @@ -0,0 +1,7841 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "This Dashboard provides a general overview of Standalone Mode", + "editable": true, + "gnetId": 2381, + "graphTooltip": 0, + "id": 6, + "iteration": 1617106562807, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 60 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Usage", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_idle" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "* -1 + 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "groupBy": [], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_idle" + ], + "type": "field" + }, + { + "params": [ + "*-1+100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 39, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.4", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "uptime" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "{{ SERVERNAME }} - System Uptime", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 0 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-elasticsearch" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-elasticsearch" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - ES CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=", + "value": "so-kibana" + }, + { + "condition": "AND", + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "container_name", + "operator": "=", + "value": "so-kibana" + }, + { + "condition": "AND", + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Kibana CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 0 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "suridrop", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "drop" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "* 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Suricata Packet Loss", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 0 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "stenodrop", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "drop" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - PCAP Packet Loss", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 73, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Used", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "disk", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/" + } + ] + }, + { + "groupBy": [], + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Disk Used(/)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 5 + }, + "hiddenSeries": false, + "id": 74, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Used", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "disk", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "used_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/nsm" + } + ] + }, + { + "groupBy": [], + "measurement": "disk", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_used_percent" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "path", + "operator": "=", + "value": "/nsm" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Disk Used(/nsm)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 5 + }, + "hiddenSeries": false, + "id": 41, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - InfluxDB CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 45, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Logstash CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 5 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Suri CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 5 + }, + "hiddenSeries": false, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Steno CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Interrupt": "#70DBED", + "Nice": "#629E51", + "SoftIRQ": "#EA6460", + "System": "#BF1B00", + "User": "#1F78C1", + "Wait": "#F2C96D", + "cpu.mean": "#629E51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 4, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "System", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_system" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "User", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_user" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Nice", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_nice" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Interrupt", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_irq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "Wait", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_iowait" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + }, + { + "alias": "SoftIRQ", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_softirq" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "cpu", + "operator": "=", + "value": "cpu-total" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "Percent(%)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "InBound": "#629E51", + "OutBound": "#5195CE", + "net.non_negative_derivative": "super-light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 10 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "net", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes_recv" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + }, + { + "alias": "Outbound", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "net", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_sent\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes_sent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + }, + { + "groupBy": [], + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + }, + { + "groupBy": [], + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_sent" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MANINT }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Management Traffic", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 10 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-suricata" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Suri Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 10 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-steno" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Steno Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "#cpu": "green", + "1 Minute Average": "#EAB839", + "15 Minute Average": "#BF1B00", + "5 Minute Average": "#E0752D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "#cpu", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "#cpu", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "n_cpus" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "1 Minute Average", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load1" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "5 Minute Average", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load5" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "15 Minute Average", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "system", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load15" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Load Average", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "InBound": "#629E51", + "OutBound": "#5195CE", + "net.non_negative_derivative": "light-orange" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 15 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "net", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT 8 * non_negative_derivative(mean(\"bytes_recv\"),1s) FROM \"net\" WHERE \"host\" = 'JumpHost' AND \"interface\" = 'eth0' AND $timeFilter GROUP BY time($interval) fill(null)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes_recv" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MONINT }}" + } + ] + }, + { + "groupBy": [], + "measurement": "net", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_bytes_recv" + ], + "type": "field" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "interface", + "operator": "=", + "value": "{{ MONINT }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Monitor Traffic", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 15 + }, + "id": 37, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.4", + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "healthcheck", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "zeek_restart" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Zeek Restarts via Healthcheck", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 2, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": null, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 259200 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 432000 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 15 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Oldest Pcap", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "pcapage", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "seconds" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "pcapage", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_seconds" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - PCAP Retention", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Blocked": "#BF1B00", + "Running": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 7, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Blocked", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "processes", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "blocked" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Running", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "processes", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "running" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Sleep", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "processes", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "sleeping" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Processes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 20 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + }, + { + "alias": "Outbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-logstash" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Logstash Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 20 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + " / {{ CPUS }}" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Zeek CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 20 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-zeek" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Zeek Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Threads", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "processes", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_threads" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Total Threads", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 25 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + }, + { + "alias": "Outbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-influxdb" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - InfluxDB Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 25 + }, + "hiddenSeries": false, + "id": 71, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "zeekcaptureloss", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "loss" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Zeek Capture Loss", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 25 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "zeekdrop", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "drop" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "* 100" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Zeek Packet Loss", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 53, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_iowait" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - IO Wait", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 30 + }, + "hiddenSeries": false, + "id": 51, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Inbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "rx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-aptcacherng" + } + ] + }, + { + "alias": "Outbound", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_net", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tx_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + }, + { + "params": [ + "*8" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-aptcacherng" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Proxy Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bps", + "label": "Bits/Sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 30 + }, + "hiddenSeries": false, + "id": 69, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "influxsize", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "kbytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "influxsize", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_kbytes" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - InfluxDB Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "deckbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 30 + }, + "hiddenSeries": false, + "id": 55, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "redisqueue", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "unparsed" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "redisqueue", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_unparsed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Redis Queue", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 35 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Read", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "diskio", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "read_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Write", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "diskio", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "write_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Disk I/O", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 35 + }, + "hiddenSeries": false, + "id": 76, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "EPS", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "esteps", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "eps" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "esteps", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_eps" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Estimated EPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "EPS", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 35 + }, + "hiddenSeries": false, + "id": 61, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_mem", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Redis Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 35 + }, + "hiddenSeries": false, + "id": 59, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "usage_percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "/ 16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] + }, + { + "groupBy": [], + "measurement": "docker_container_cpu", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_usage_percent" + ], + "type": "field" + }, + { + "params": [ + "/16" + ], + "type": "math" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + }, + { + "condition": "AND", + "key": "container_name", + "operator": "=", + "value": "so-redis" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Redis CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Buffered": "#6ED0E0", + "Cached": "#F9934E", + "Free": "#629E51", + "Used": "#58140C" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "decimals": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 6, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Used", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "used" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Buffered", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffered" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Cached", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "cached" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Free", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "free" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Memory", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "InfluxDB", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 40 + }, + "id": 33, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "docs_count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": "", + "title": "{{ SERVERNAME }} - ES Documents", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 40 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "store_size_in_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_store_size_in_bytes" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - ES Store Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 40 + }, + "hiddenSeries": false, + "id": 65, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "elasticsearch_jvm", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "threads_count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "elasticsearch_jvm", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_threads_count" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - ES Thread Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 40 + }, + "hiddenSeries": false, + "id": 63, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "fielddata_memory_size_in_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "groupBy": [], + "measurement": "elasticsearch_indices", + "orderByTime": "ASC", + "policy": "so_long_term", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "mean_fielddata_memory_size_in_bytes" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - ES Fielddata Cache Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 67, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Total", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + }, + { + "alias": "Used", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "mem", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "used" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "{{ SERVERNAME }}" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "{{ SERVERNAME }} - Memory(Used)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "10s", + "value": "10s" + }, + "error": null, + "hide": 0, + "label": null, + "name": "Interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_Interval" + }, + { + "selected": true, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "10s, 1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Standalone Mode - {{ SERVERNAME }} Overview", + "uid": "so_overview", + "version": 1 + } \ No newline at end of file From 4f3b3a787c5cf9348518280193263b69aacf314b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 1 Apr 2021 10:18:01 -0400 Subject: [PATCH 20/76] change defaults for testing, remove measurements list since cq uses wildcard now - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- salt/influxdb/defaults.yaml | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 927cd7b52..ce5624be2 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,34 +2,12 @@ influxdb: retention_policies: autogen: default: True - duration: 1h + duration: 2d shard_duration: 1h so_long_term: default: False - duration: 2d + duration: 7d shard_duration: 1d downsample: so_long_term: - resolution: 5m - measurements: - - cpu - - disk - - diskio - - docker_container_cpu - - docker_container_mem - - docker_container_net - - elasticsearch_indices - - elasticsearch_jvm - - esteps - - healthcheck - - influxsize - - mem - - net - - pcapage - - processes - - redisqueue - - stenodrop - - suridrop - - system - - zeekcaptureloss - - zeekdrop \ No newline at end of file + resolution: 5m \ No newline at end of file From 5cda35db0aa0d8d9b6f2ded8a9a4ef959550ab58 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 1 Apr 2021 10:45:54 -0400 Subject: [PATCH 21/76] change defaults for testing - https://github.com/Security-Onion-Solutions/securityonion/issues/3264 --- salt/influxdb/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index ce5624be2..03ac4c3c6 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,7 +2,7 @@ influxdb: retention_policies: autogen: default: True - duration: 2d + duration: 1d shard_duration: 1h so_long_term: default: False From 8eef574342f27c46d581331114cd2d4dbfd5d027 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 08:28:01 -0400 Subject: [PATCH 22/76] install python3-influxdb and create requires --- salt/influxdb/init.sls | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 1b21612b3..67342f826 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -13,6 +13,10 @@ include: - salt.minion +python3-influxdb: + pkg.installed: + - name: python3-influxdb + # Influx DB influxconfdir: file.directory: @@ -62,6 +66,7 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb + telegraf_database: influxdb_database.present: - name: telegraf @@ -72,6 +77,7 @@ telegraf_database: - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb + - pkg: python3-influxdb {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: @@ -90,28 +96,9 @@ telegraf_database: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_retention_policy.present_patch + - pkg: python3-influxdb {% endfor %} -{# -{% for dest_rp in influxdb.downsample.keys() %} - {% for measurement in influxdb.downsample[dest_rp].measurements %} -so_downsample_{{measurement}}_cq: - influxdb_continuous_query.present: - - name: so_downsample_{{measurement}}_cq - - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}), * - - ssl: True - - verify_ssl: /etc/pki/ca.crt - - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] - - influxdb_host: {{ MANAGER }} - - require: - - docker_container: so-influxdb - - influxdb_database: telegraf_database - - file: influxdb_continuous_query.present_patch - {% endfor %} -{% endfor %} -#} - {% for dest_rp in influxdb.downsample.keys() %} so_downsample_cq: influxdb_continuous_query.present: @@ -126,6 +113,7 @@ so_downsample_cq: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch + - pkg: python3-influxdb {% endfor %} {% endif %} From d792c65ce326e246ae603fcf3b82dd162da4ac60 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 09:25:25 -0400 Subject: [PATCH 23/76] change how influx is patch and python3-influxdb is installed --- salt/influxdb/init.sls | 11 ++++------- salt/salt/minion.sls | 20 +------------------- salt/salt/python3-influxdb.sls | 23 +++++++++++++++++++++++ 3 files changed, 28 insertions(+), 26 deletions(-) create mode 100644 salt/salt/python3-influxdb.sls diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 67342f826..55f350099 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -12,10 +12,7 @@ include: - salt.minion - -python3-influxdb: - pkg.installed: - - name: python3-influxdb + - salt.python3-influxdb # Influx DB influxconfdir: @@ -77,7 +74,7 @@ telegraf_database: - influxdb_host: {{ MANAGER }} - require: - docker_container: so-influxdb - - pkg: python3-influxdb + - sls: salt.python3-influxdb {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: @@ -96,7 +93,7 @@ telegraf_database: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_retention_policy.present_patch - - pkg: python3-influxdb + - sls: salt.python3-influxdb {% endfor %} {% for dest_rp in influxdb.downsample.keys() %} @@ -113,7 +110,7 @@ so_downsample_cq: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - - pkg: python3-influxdb + - sls: python3-influxdb {% endfor %} {% endif %} diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 2ac66c81a..41f9bfaf2 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -67,22 +67,4 @@ salt_minion_service: patch_pkg: pkg.installed: - - name: patch - - -#https://github.com/saltstack/salt/issues/59766 -influxdb_continuous_query.present_patch: - file.patch: - - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_continuous_query.py - - source: salt://salt/files/influxdb_continuous_query.py.patch - -#https://github.com/saltstack/salt/issues/59761 -influxdb_retention_policy.present_patch: - file.patch: - - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_retention_policy.py - - source: salt://salt/files/influxdb_retention_policy.py.patch - -influxdbmod.py_shard_duration_patch: - file.patch: - - name: /usr/lib/python3.6/site-packages/salt/modules/influxdbmod.py - - source: salt://salt/files/influxdbmod.py.patch \ No newline at end of file + - name: patch \ No newline at end of file diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls new file mode 100644 index 000000000..0f8cad42e --- /dev/null +++ b/salt/salt/python3-influxdb.sls @@ -0,0 +1,23 @@ +python3_influxdb: + pkg.installed: + - name: python3-influxdb + +#https://github.com/saltstack/salt/issues/59766 +influxdb_continuous_query.present_patch: + file.patch: + - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_continuous_query.py + - source: salt://salt/files/influxdb_continuous_query.py.patch + - pkg: python3_influxdb + +#https://github.com/saltstack/salt/issues/59761 +influxdb_retention_policy.present_patch: + file.patch: + - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_retention_policy.py + - source: salt://salt/files/influxdb_retention_policy.py.patch + - pkg: python3_influxdb + +influxdbmod.py_shard_duration_patch: + file.patch: + - name: /usr/lib/python3.6/site-packages/salt/modules/influxdbmod.py + - source: salt://salt/files/influxdbmod.py.patch + - pkg: python3_influxdb \ No newline at end of file From af86a9dac01fe511677d72718f1ac30675a79fe1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 09:52:22 -0400 Subject: [PATCH 24/76] handle different paths for salt states/modules based on os --- salt/salt/map.jinja | 4 ++++ salt/salt/python3-influxdb.sls | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 5d6d980be..489a41dd2 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -4,9 +4,13 @@ {% if grains.os == 'Ubuntu' %} {% set SPLITCHAR = '+' %} {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep salt-* ; echo $?', python_shell=True) %} + {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} + {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} + {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} + {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 0f8cad42e..f0536e36f 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -1,3 +1,6 @@ +{% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %} +{% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} + python3_influxdb: pkg.installed: - name: python3-influxdb @@ -5,19 +8,19 @@ python3_influxdb: #https://github.com/saltstack/salt/issues/59766 influxdb_continuous_query.present_patch: file.patch: - - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_continuous_query.py + - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - pkg: python3_influxdb #https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - - name: /usr/lib/python3.6/site-packages/salt/states/influxdb_retention_policy.py + - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - pkg: python3_influxdb influxdbmod.py_shard_duration_patch: file.patch: - - name: /usr/lib/python3.6/site-packages/salt/modules/influxdbmod.py + - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - pkg: python3_influxdb \ No newline at end of file From b53017ee87603f1d15c441bb4913a65b2989fd1e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 10:44:46 -0400 Subject: [PATCH 25/76] ensure salt python modules are installed and patched during soup --- salt/common/tools/sbin/soup | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 2a1ddab1c..2d9e0a146 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -729,6 +729,13 @@ echo "" echo "Starting Salt Master service." systemctl start salt-master +sleep 10 + +echo "" +echo "Ensuring python modules for Salt are installed and patched." +salt-call state.apply python3-influxdb -l info queue=True +echo "" + # Only regenerate osquery packages if Fleet is enabled FLEET_MANAGER=$(lookup_pillar fleet_manager) FLEET_NODE=$(lookup_pillar fleet_node) From 0a01d7b0412e14b9d16bb9e317bc5437504cc4a4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 11:14:13 -0400 Subject: [PATCH 26/76] fix var --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 97e61e6e2..febdd4a6c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -167,7 +167,7 @@ gpg_rpm_import() { if [[ "$WHATWOULDYOUSAYYAHDOHERE" == "setup" ]]; then local RPMKEYSLOC="../salt/repo/client/files/centos/keys" else - local RPMKEYSLOC="$UPDATEDIR/salt/repo/client/files/centos/keys" + local RPMKEYSLOC="$UPDATE_DIR/salt/repo/client/files/centos/keys" fi RPMKEYS=('RPM-GPG-KEY-EPEL-7' 'GPG-KEY-WAZUH' 'docker.pub' 'SALTSTACK-GPG-KEY.pub' 'securityonion.pub') From 1f654d44440fd1431d911bda96c67ea1db1ba663 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 12:04:58 -0400 Subject: [PATCH 27/76] fix the state apply --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 2d9e0a146..05c56008a 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -733,7 +733,7 @@ sleep 10 echo "" echo "Ensuring python modules for Salt are installed and patched." -salt-call state.apply python3-influxdb -l info queue=True +salt-call state.apply salt.python3-influxdb -l info queue=True echo "" # Only regenerate osquery packages if Fleet is enabled From 17c95723ec6670b94fb1f03518ffa2b11a35a03e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 16:44:40 -0400 Subject: [PATCH 28/76] update package name --- salt/salt/python3-influxdb.sls | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index f0536e36f..41989816f 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -1,26 +1,26 @@ {% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %} {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} -python3_influxdb: +securityonion_python3_influxdb: pkg.installed: - - name: python3-influxdb + - name: securityonion-python3-influxdb #https://github.com/saltstack/salt/issues/59766 influxdb_continuous_query.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - - pkg: python3_influxdb + - pkg: securityonion_python3_influxdb #https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - - pkg: python3_influxdb + - pkg: securityonion_python3_influxdb influxdbmod.py_shard_duration_patch: file.patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - - pkg: python3_influxdb \ No newline at end of file + - pkg: securityonion_python3_influxdb \ No newline at end of file From c9298137b52c1466f087246de153d553d36d6d04 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 17:08:35 -0400 Subject: [PATCH 29/76] adding docker-ce.repo to delete list --- salt/repo/client/map.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/repo/client/map.jinja b/salt/repo/client/map.jinja index ccfa1eae2..dedb96783 100644 --- a/salt/repo/client/map.jinja +++ b/salt/repo/client/map.jinja @@ -10,6 +10,7 @@ 'CentOS-Sources.repo', 'CentOS-Vault.repo', 'CentOS-x86_64-kernel.repo', + 'docker-ce.repo', 'epel.repo', 'epel-testing.repo', 'saltstack.repo', From a7247e98121f8dc1ee13760efc4403be3ae8a83b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 17:17:49 -0400 Subject: [PATCH 30/76] update package name --- salt/salt/map.jinja | 2 ++ salt/salt/python3-influxdb.sls | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 489a41dd2..fb6cadb1b 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -6,11 +6,13 @@ {% set SALTNOTHELD = salt['cmd.run']('apt-mark showhold | grep salt-* ; echo $?', python_shell=True) %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} + {% set PYTHON3INFLUX= 'python3-influxdb' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} + {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 41989816f..36347cb95 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -1,26 +1,27 @@ {% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %} {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} +{% from "salt/map.jinja" import PYTHON3INFLUX with context %} -securityonion_python3_influxdb: +python3_influxdb: pkg.installed: - - name: securityonion-python3-influxdb + - name: {{ PYTHON3INFLUX }} #https://github.com/saltstack/salt/issues/59766 influxdb_continuous_query.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - - pkg: securityonion_python3_influxdb + - pkg: python3_influxdb #https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - - pkg: securityonion_python3_influxdb + - pkg: python3_influxdb influxdbmod.py_shard_duration_patch: file.patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - - pkg: securityonion_python3_influxdb \ No newline at end of file + - pkg: python3_influxdb \ No newline at end of file From 5cedf98f555d79c1b80276ebb31ce125f5073a66 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 18:05:52 -0400 Subject: [PATCH 31/76] change path --- salt/salt/map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index fb6cadb1b..67ab4a594 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -10,8 +10,8 @@ {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} - {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} - {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} + {% set SALT_STATE_CODE_PATH = '/usr/local/lib/python3.6/site-packages/salt/states' %} + {% set SALT_MODULE_CODE_PATH = '/usr/local/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% endif %} From 21077ef26e09c2f1a0578d24e13ee23d01db05d1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 18:09:11 -0400 Subject: [PATCH 32/76] undo path change --- salt/salt/map.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 67ab4a594..fb6cadb1b 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -10,8 +10,8 @@ {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep salt-* ; echo $?', python_shell=True) %} - {% set SALT_STATE_CODE_PATH = '/usr/local/lib/python3.6/site-packages/salt/states' %} - {% set SALT_MODULE_CODE_PATH = '/usr/local/lib/python3.6/site-packages/salt/modules' %} + {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} + {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% endif %} From f5ddb084b6f850f23652432a85fe60254f3169c3 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 18:22:44 -0400 Subject: [PATCH 33/76] create salt.helper-packages state, use it to install the patch package --- salt/common/init.sls | 2 -- salt/salt/helper-packages.sls | 3 +++ salt/salt/minion.sls | 1 + salt/salt/python3-influxdb.sls | 15 ++++++++++++--- 4 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 salt/salt/helper-packages.sls diff --git a/salt/common/init.sls b/salt/common/init.sls index 6d2a9aea7..cb6039756 100644 --- a/salt/common/init.sls +++ b/salt/common/init.sls @@ -95,7 +95,6 @@ commonpkgs: - python3-mysqldb - python3-packaging - git - - patch heldpackages: pkg.installed: @@ -135,7 +134,6 @@ commonpkgs: - lvm2 - openssl - git - - patch heldpackages: pkg.installed: diff --git a/salt/salt/helper-packages.sls b/salt/salt/helper-packages.sls new file mode 100644 index 000000000..93ad76a22 --- /dev/null +++ b/salt/salt/helper-packages.sls @@ -0,0 +1,3 @@ +patch_package: + pkg.installed: + - name: patch \ No newline at end of file diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 41f9bfaf2..0dc8760cb 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -7,6 +7,7 @@ include: - salt + - salt.helper_packages - systemd.reload {% if INSTALLEDSALTVERSION|string != SALTVERSION|string %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 36347cb95..b939c3380 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -2,6 +2,9 @@ {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} {% from "salt/map.jinja" import PYTHON3INFLUX with context %} +include: + - salt.helper-packages + python3_influxdb: pkg.installed: - name: {{ PYTHON3INFLUX }} @@ -11,17 +14,23 @@ influxdb_continuous_query.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - - pkg: python3_influxdb + - require: + - pkg: python3_influxdb + - pkg: patch_package #https://github.com/saltstack/salt/issues/59761 influxdb_retention_policy.present_patch: file.patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - - pkg: python3_influxdb + - require: + - pkg: python3_influxdb + - pkg: patch_package influxdbmod.py_shard_duration_patch: file.patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - - pkg: python3_influxdb \ No newline at end of file + - require: + - pkg: python3_influxdb + - pkg: patch_package \ No newline at end of file From 1fbf77d09067ca71c5b913d948cf714bf60904e9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 18:53:00 -0400 Subject: [PATCH 34/76] fix state name --- salt/salt/minion.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/minion.sls b/salt/salt/minion.sls index 0dc8760cb..2f4c3d2ac 100644 --- a/salt/salt/minion.sls +++ b/salt/salt/minion.sls @@ -7,7 +7,7 @@ include: - salt - - salt.helper_packages + - salt.helper-packages - systemd.reload {% if INSTALLEDSALTVERSION|string != SALTVERSION|string %} From 28982e0e0b9fa2cec0f8bd4f1ea734f99551da5a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 21 Apr 2021 19:22:07 -0400 Subject: [PATCH 35/76] fix requirement --- salt/influxdb/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 55f350099..8248c6228 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -110,7 +110,7 @@ so_downsample_cq: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - - sls: python3-influxdb + - sls: salt.python3-influxdb {% endfor %} {% endif %} From 5584c4f1aee056a0d8946bb74776157a1f71a859 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 28 Apr 2021 18:01:33 -0400 Subject: [PATCH 36/76] define and install the dependencies for python36-influxdb --- salt/salt/map.jinja | 1 + salt/salt/python3-influxdb.sls | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 21fcb9a58..a4289a6b8 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -15,6 +15,7 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} + {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index b939c3380..424f06493 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -1,10 +1,15 @@ {% from "salt/map.jinja" import SALT_STATE_CODE_PATH with context %} {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} {% from "salt/map.jinja" import PYTHON3INFLUX with context %} +{% from "salt/map.jinja" import PYTHON3INFLUXDEPS with context %} include: - salt.helper-packages +python3_influxdb_dependencies: + pkg.installed: + - pkgs: {{ PYTHON3INFLUXDEPS }} + python3_influxdb: pkg.installed: - name: {{ PYTHON3INFLUX }} From 454b541a2e02e8eddc8182726059ea60eb1eea79 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 28 Apr 2021 18:04:14 -0400 Subject: [PATCH 37/76] merge with dev, change version so test box doesnt try to upgrade to 2.3.60 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 678d59d4f..58e1f4fa9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.60 +2.3.50 From f8e01d5d53d83b3cf3221037abd2f18532c8985a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 06:43:05 -0400 Subject: [PATCH 38/76] let the state retry incase influxdb isnt fully up yet --- salt/influxdb/init.sls | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 8248c6228..22e2c9856 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -75,6 +75,10 @@ telegraf_database: - require: - docker_container: so-influxdb - sls: salt.python3-influxdb + - timeout: 30 + - retry: + attempts: 5 + interval: 30 {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: From 907dbe638886e37947714e74d935270a6f9dde84 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 08:47:33 -0400 Subject: [PATCH 39/76] for for influx to be up so the reliant states dont fail --- salt/influxdb/init.sls | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 22e2c9856..609263bb5 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -63,6 +63,16 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb +wait_for_influxdb: + http.query: + - name: 'https://{{MANAGER}}:8086/query?q=SHOW+DATABASES' + - ssl: True + - verify_ssl: False + - status: 200 + - timeout: 30 + - retry: + attempts: 5 + interval: 60 telegraf_database: influxdb_database.present: @@ -75,10 +85,7 @@ telegraf_database: - require: - docker_container: so-influxdb - sls: salt.python3-influxdb - - timeout: 30 - - retry: - attempts: 5 - interval: 30 + - http: wait_for_influxdb {% for rp in influxdb.retention_policies.keys() %} {{rp}}_retention_policy: From 93ee96b1cd224176f1e5e107cd3839e186b9a4ac Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 10:19:42 -0400 Subject: [PATCH 40/76] Ignore "Status .* was not found" due to output from salt http.query or http.wait_for_successful_query states used with retry --- setup/so-setup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/so-setup b/setup/so-setup index d7dd3b660..2160929cf 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -918,10 +918,11 @@ success=$(tail -10 $setup_log | grep Failed | awk '{ print $2}') if [[ $success != 0 ]]; then SO_ERROR=1; fi # Check entire setup log for errors or unexpected salt states and ensure cron jobs are not reporting errors to root's mailbox -if grep -q -E "ERROR|Result: False" $setup_log || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then +# Ignore "Status .* was not found" due to output from salt http.query or http.wait_for_successful_query states used with retry +if grep -q -E "ERROR|Result: False" $setup_log | grep -qvE "Status .* was not found" || [[ -s /var/spool/mail/root && "$setup_type" == "iso" ]]; then SO_ERROR=1 - grep --color=never "ERROR" "$setup_log" > "$error_log" + grep --color=never "ERROR" "$setup_log" | grep -qvE "Status .* was not found" > "$error_log" fi if [[ -n $SO_ERROR ]]; then From 450a01784befcb723409be8e293905f134c4d1c8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:22:31 -0400 Subject: [PATCH 41/76] support installing via pip for ubuntu --- salt/salt/map.jinja | 7 +++++-- salt/salt/python3-influxdb.sls | 10 +++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index a4289a6b8..8a3c26bb8 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -7,7 +7,9 @@ {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} - {% set PYTHON3INFLUX= 'python3-influxdb' %} + {% set PYTHON3INFLUX= 'influxdb == 5.3.1' %} + {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'dateutil', 'pytz', 'requests'] + {% set PYTHONINSTALLER = 'pip' %} {% else %} {% set SPLITCHAR = '-' %} {% set SALTNOTHELD = salt['cmd.run']('yum versionlock list | grep -q salt ; echo $?', python_shell=True) %} @@ -15,7 +17,8 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} - {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} + {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] + {% set PYTHONINSTALLER = 'pkg' %} {% endif %} {% set INSTALLEDSALTVERSION = salt['pkg.version']('salt-minion').split(SPLITCHAR)[0] %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 424f06493..67fde3be2 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -2,16 +2,19 @@ {% from "salt/map.jinja" import SALT_MODULE_CODE_PATH with context %} {% from "salt/map.jinja" import PYTHON3INFLUX with context %} {% from "salt/map.jinja" import PYTHON3INFLUXDEPS with context %} +{% from "salt/map.jinja" import PYTHONINSTALLER with context %} include: - salt.helper-packages +{# python3_influxdb_dependencies: - pkg.installed: + {{PYTHONINSTALLER}}.installed: - pkgs: {{ PYTHON3INFLUXDEPS }} +#} python3_influxdb: - pkg.installed: + {{PYTHONINSTALLER}}.installed: - name: {{ PYTHON3INFLUX }} #https://github.com/saltstack/salt/issues/59766 @@ -38,4 +41,5 @@ influxdbmod.py_shard_duration_patch: - source: salt://salt/files/influxdbmod.py.patch - require: - pkg: python3_influxdb - - pkg: patch_package \ No newline at end of file + - pkg: patch_package + - reload_modules: True \ No newline at end of file From e7660d68cb50827b9ed6901b16c4adcb0d7661bc Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:25:29 -0400 Subject: [PATCH 42/76] add %} --- salt/salt/map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 8a3c26bb8..9cf9331f5 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -8,7 +8,7 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} {% set PYTHON3INFLUX= 'influxdb == 5.3.1' %} - {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'dateutil', 'pytz', 'requests'] + {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} {% else %} {% set SPLITCHAR = '-' %} From 888d637b67e6444edb012c57d8a3531b36d537ef Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:26:24 -0400 Subject: [PATCH 43/76] add %} --- salt/salt/map.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 9cf9331f5..9b84b72e2 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -17,7 +17,7 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} - {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] + {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% set PYTHONINSTALLER = 'pkg' %} {% endif %} From 689a01423fea65468f4b592052c66bdf7eab37c7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:28:31 -0400 Subject: [PATCH 44/76] fix deps --- salt/salt/python3-influxdb.sls | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 67fde3be2..3cad07144 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -23,7 +23,7 @@ influxdb_continuous_query.present_patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_continuous_query.py - source: salt://salt/files/influxdb_continuous_query.py.patch - require: - - pkg: python3_influxdb + - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package #https://github.com/saltstack/salt/issues/59761 @@ -32,7 +32,7 @@ influxdb_retention_policy.present_patch: - name: {{ SALT_STATE_CODE_PATH }}/influxdb_retention_policy.py - source: salt://salt/files/influxdb_retention_policy.py.patch - require: - - pkg: python3_influxdb + - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package influxdbmod.py_shard_duration_patch: @@ -40,6 +40,6 @@ influxdbmod.py_shard_duration_patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - require: - - pkg: python3_influxdb + - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package - reload_modules: True \ No newline at end of file From d475e50bef82df40fb59ce15dc087fe088abc314 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 29 Apr 2021 13:49:15 -0400 Subject: [PATCH 45/76] add deps for ubuntu --- salt/salt/map.jinja | 2 +- salt/salt/python3-influxdb.sls | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 9b84b72e2..5b211e969 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -8,7 +8,7 @@ {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} {% set PYTHON3INFLUX= 'influxdb == 5.3.1' %} - {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'dateutil', 'pytz', 'requests'] %} + {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} {% else %} {% set SPLITCHAR = '-' %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index 3cad07144..f65a0c6b4 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -7,11 +7,9 @@ include: - salt.helper-packages -{# python3_influxdb_dependencies: {{PYTHONINSTALLER}}.installed: - pkgs: {{ PYTHON3INFLUXDEPS }} -#} python3_influxdb: {{PYTHONINSTALLER}}.installed: From e32ca284c52c86aee3274bcf0940f6c6d26920c7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 30 Apr 2021 11:10:31 -0400 Subject: [PATCH 46/76] ensure proper version of python3-influxdb is installed prior to running the states that require it --- salt/influxdb/init.sls | 6 ++++++ salt/salt/python3-influxdb.sls | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 609263bb5..cda095a31 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -7,6 +7,9 @@ {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %} {% import_yaml 'influxdb/defaults.yaml' as default_settings %} {% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} +{% from 'salt/map.jinja' import PYTHON3INFLUX with context %} +{% set PYTHONINFLUXVERSION = PYTHON3INFLUX.split("==")[1] | trim %} +{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c 'import influxdb; print (influxdb.__version__)'", python_shell=True) | trim %} {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} @@ -63,6 +66,8 @@ append_so-influxdb_so-status.conf: - name: /opt/so/conf/so-status/so-status.conf - text: so-influxdb +# We have to make sure the influxdb module is the right version prior to state run since reload_modules is bugged +{% if PYTHONINFLUXVERSIONINSTALLED == PYTHONINFLUXVERSION %} wait_for_influxdb: http.query: - name: 'https://{{MANAGER}}:8086/query?q=SHOW+DATABASES' @@ -124,6 +129,7 @@ so_downsample_cq: - sls: salt.python3-influxdb {% endfor %} +{% endif %} {% endif %} {% else %} diff --git a/salt/salt/python3-influxdb.sls b/salt/salt/python3-influxdb.sls index f65a0c6b4..acadf2161 100644 --- a/salt/salt/python3-influxdb.sls +++ b/salt/salt/python3-influxdb.sls @@ -33,11 +33,12 @@ influxdb_retention_policy.present_patch: - {{PYTHONINSTALLER}}: python3_influxdb - pkg: patch_package +# We should be able to set reload_modules: True in this state in order to tell salt to reload its python modules due to us possibly installing +# and possibly modifying modules in this state. This is bugged according to https://github.com/saltstack/salt/issues/24925 influxdbmod.py_shard_duration_patch: file.patch: - name: {{ SALT_MODULE_CODE_PATH }}/influxdbmod.py - source: salt://salt/files/influxdbmod.py.patch - require: - {{PYTHONINSTALLER}}: python3_influxdb - - pkg: patch_package - - reload_modules: True \ No newline at end of file + - pkg: patch_package \ No newline at end of file From 038cadeae892c6efdb824cb6274eaee8bf4b4924 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 30 Apr 2021 12:31:57 -0400 Subject: [PATCH 47/76] change version to 2.3.60 to prepare to push to dev --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 58e1f4fa9..678d59d4f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.50 +2.3.60 From d6b9154a88368e710cd5683036b3ec76db4d6dfd Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 30 Apr 2021 14:48:51 -0400 Subject: [PATCH 48/76] change how version to be installed is defined to work with centos --- salt/influxdb/init.sls | 4 ++-- salt/salt/map.jinja | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index cda095a31..c6120d292 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -8,8 +8,8 @@ {% import_yaml 'influxdb/defaults.yaml' as default_settings %} {% set influxdb = salt['grains.filter_by'](default_settings, default='influxdb', merge=salt['pillar.get']('influxdb', {})) %} {% from 'salt/map.jinja' import PYTHON3INFLUX with context %} -{% set PYTHONINFLUXVERSION = PYTHON3INFLUX.split("==")[1] | trim %} -{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c 'import influxdb; print (influxdb.__version__)'", python_shell=True) | trim %} +{% from 'salt/map.jinja' import PYTHONINFLUXVERSION with context %} +{% set PYTHONINFLUXVERSIONINSTALLED = salt['cmd.run']("python3 -c 'import influxdb; print (influxdb.__version__)'", python_shell=True) %} {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-eval', 'so-standalone'] and GRAFANA == 1 %} diff --git a/salt/salt/map.jinja b/salt/salt/map.jinja index 5b211e969..b0b9ffb2a 100644 --- a/salt/salt/map.jinja +++ b/salt/salt/map.jinja @@ -7,7 +7,8 @@ {% set SALTPACKAGES = ['salt-common', 'salt-master', 'salt-minion'] %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3/dist-packages/salt/modules' %} - {% set PYTHON3INFLUX= 'influxdb == 5.3.1' %} + {% set PYTHONINFLUXVERSION = '5.3.1' %} + {% set PYTHON3INFLUX= 'influxdb == ' ~ PYTHONINFLUXVERSION %} {% set PYTHON3INFLUXDEPS= ['certifi', 'chardet', 'python-dateutil', 'pytz', 'requests'] %} {% set PYTHONINSTALLER = 'pip' %} {% else %} @@ -16,6 +17,7 @@ {% set SALTPACKAGES = ['salt', 'salt-master', 'salt-minion'] %} {% set SALT_STATE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/states' %} {% set SALT_MODULE_CODE_PATH = '/usr/lib/python3.6/site-packages/salt/modules' %} + {% set PYTHONINFLUXVERSION = '5.3.1' %} {% set PYTHON3INFLUX= 'securityonion-python3-influxdb' %} {% set PYTHON3INFLUXDEPS= ['python36-certifi', 'python36-chardet', 'python36-dateutil', 'python36-pytz', 'python36-requests'] %} {% set PYTHONINSTALLER = 'pkg' %} From 12306368cfe6125869d32319ef471d76856c2aee Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 08:37:52 -0400 Subject: [PATCH 49/76] add post upgrade function for 2.3.60 soup to migrate influxdb data --- salt/common/tools/sbin/soup | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b3484446f..395ebef36 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -284,6 +284,7 @@ postupgrade_changes() { [[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2 [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30 [[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40 + [[ "$POSTVERSION" == 2.3.50 ]] && post_2.3.5X_to_2.3.60 } post_rc1_to_2.3.21() { @@ -304,6 +305,15 @@ post_2.3.30_to_2.3.40() { POSTVERSION=2.3.40 } +post_2.3.5X_to_2.3.60() { + # These are post upgrade changes that need to take place after all the new states have been applied. + echo "" + echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes." + docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$SOUP_LOG" 2>&1 & + + POSTVERSION=2.3.60 +} + rc1_to_rc2() { From 9b065155f46bd609ccf24e9b818a0219ed00d728 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 08:56:13 -0400 Subject: [PATCH 50/76] log iunfluxdb migration to new log --- salt/common/tools/sbin/soup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 395ebef36..b1e326c73 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -24,6 +24,7 @@ INSTALLEDSALTVERSION=$(salt --versions-report | grep Salt: | awk {'print $2'}) DEFAULT_SALT_DIR=/opt/so/saltstack/default BATCHSIZE=5 SOUP_LOG=/root/soup.log +INFLUXDB_MIGRATION_LOG=/opt/so/log/influxdb/soup_migration.log WHATWOULDYOUSAYYAHDOHERE=soup add_common() { @@ -309,7 +310,7 @@ post_2.3.5X_to_2.3.60() { # These are post upgrade changes that need to take place after all the new states have been applied. echo "" echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes." - docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$SOUP_LOG" 2>&1 & + docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 & POSTVERSION=2.3.60 } From f186a3dde923ce29fe9eccd7a7cd0e451d4c65b4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 09:30:38 -0400 Subject: [PATCH 51/76] make sure user sees influxdb migration message by requiring enter to be pressed to continue --- salt/common/tools/sbin/soup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b1e326c73..b08271e1f 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -309,7 +309,9 @@ post_2.3.30_to_2.3.40() { post_2.3.5X_to_2.3.60() { # These are post upgrade changes that need to take place after all the new states have been applied. echo "" - echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes." + echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." + echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" + read -p "Press enter to continue." docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 & POSTVERSION=2.3.60 From e2f52765e4ae48e851ddad7675df6239906c43c5 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 09:34:42 -0400 Subject: [PATCH 52/76] add newline --- salt/common/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index b08271e1f..de968caea 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -312,6 +312,7 @@ post_2.3.5X_to_2.3.60() { echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" read -p "Press enter to continue." + echo "" docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 & POSTVERSION=2.3.60 From a0f183916237be25782ea7c1ed8534cb84a052e4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 09:59:16 -0400 Subject: [PATCH 53/76] run in background --- salt/common/tools/sbin/soup | 2 +- salt/influxdb/defaults.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index de968caea..4c82e48c7 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -313,7 +313,7 @@ post_2.3.5X_to_2.3.60() { echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" read -p "Press enter to continue." echo "" - docker exec -t so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 & + docker exec -dt so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 POSTVERSION=2.3.60 } diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 03ac4c3c6..54e175024 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -2,12 +2,12 @@ influxdb: retention_policies: autogen: default: True - duration: 1d - shard_duration: 1h + duration: 30d + shard_duration: 1d so_long_term: default: False - duration: 7d - shard_duration: 1d + duration: 0s + shard_duration: 7d downsample: so_long_term: resolution: 5m \ No newline at end of file From 2d91e509faecf7f32bb22b808e2c5817a3248bb7 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 10:34:13 -0400 Subject: [PATCH 54/76] update wording --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 4c82e48c7..0e6f84a63 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -309,7 +309,7 @@ post_2.3.30_to_2.3.40() { post_2.3.5X_to_2.3.60() { # These are post upgrade changes that need to take place after all the new states have been applied. echo "" - echo "Migrating InfluxDB data in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." + echo "Migration of InfluxDB data will begin in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" read -p "Press enter to continue." echo "" From 3dcaa1f6fb8494d08935ceecec14d8e7972b0c32 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 10:37:25 -0400 Subject: [PATCH 55/76] more logging for influxdb migration --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 0e6f84a63..07a9804b2 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -313,7 +313,7 @@ post_2.3.5X_to_2.3.60() { echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" read -p "Press enter to continue." echo "" - docker exec -dt so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*"' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 + docker exec -dt so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*" && echo "InfluxDB migration complete." || echo "InfluxDB migration failed."' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 POSTVERSION=2.3.60 } From fcd1bea4a3b25b1b74c28363769d8256bca3a892 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 12:06:03 -0400 Subject: [PATCH 56/76] remove auto data migration, change duration from 0s to 0d --- salt/common/tools/sbin/soup | 8 -------- salt/influxdb/defaults.yaml | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 07a9804b2..1f6a6e5d7 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -307,14 +307,6 @@ post_2.3.30_to_2.3.40() { } post_2.3.5X_to_2.3.60() { - # These are post upgrade changes that need to take place after all the new states have been applied. - echo "" - echo "Migration of InfluxDB data will begin in the background. Depending on how much data is present, the process may take an hour or two to complete after soup finishes. Do not restart the so-influxdb container or server until this completes." - echo "The status can be checked in $INFLUXDB_MIGRATION_LOG" - read -p "Press enter to continue." - echo "" - docker exec -dt so-influxdb /bin/bash -c 'influx -ssl -unsafeSsl -database telegraf -execute "SELECT mean(*) INTO "so_long_term".:MEASUREMENT FROM /.*/ GROUP BY time(5m),*" && echo "InfluxDB migration complete." || echo "InfluxDB migration failed."' >> "$INFLUXDB_MIGRATION_LOG" 2>&1 - POSTVERSION=2.3.60 } diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 54e175024..0c9a09080 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -6,7 +6,7 @@ influxdb: shard_duration: 1d so_long_term: default: False - duration: 0s + duration: 0d shard_duration: 7d downsample: so_long_term: From c5f99b012e6e1995376d5c4deb603911c623d4bb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 4 May 2021 13:58:53 -0400 Subject: [PATCH 57/76] comment out creation of cq to test data migration --- salt/influxdb/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index c6120d292..9a94b1c95 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -112,7 +112,7 @@ telegraf_database: - sls: salt.python3-influxdb {% endfor %} -{% for dest_rp in influxdb.downsample.keys() %} +{#% for dest_rp in influxdb.downsample.keys() %} so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq @@ -127,7 +127,7 @@ so_downsample_cq: - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - sls: salt.python3-influxdb -{% endfor %} +{% endfor %#} {% endif %} {% endif %} From 6a639edb052e328307a6bc90ec55041c486a8d4a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 08:33:31 -0400 Subject: [PATCH 58/76] have cq created once again --- salt/influxdb/init.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 9a94b1c95..c6120d292 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -112,7 +112,7 @@ telegraf_database: - sls: salt.python3-influxdb {% endfor %} -{#% for dest_rp in influxdb.downsample.keys() %} +{% for dest_rp in influxdb.downsample.keys() %} so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq @@ -127,7 +127,7 @@ so_downsample_cq: - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - sls: salt.python3-influxdb -{% endfor %#} +{% endfor %} {% endif %} {% endif %} From ce24781446c1ddbf736702dae33fb0d154fe7aeb Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 13:29:24 -0400 Subject: [PATCH 59/76] first take at so-infludb-clean --- salt/common/tools/so-influxdb-clear | 42 +++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 salt/common/tools/so-influxdb-clear diff --git a/salt/common/tools/so-influxdb-clear b/salt/common/tools/so-influxdb-clear new file mode 100644 index 000000000..38ef6491f --- /dev/null +++ b/salt/common/tools/so-influxdb-clear @@ -0,0 +1,42 @@ + +echo "This script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." +echo "The duration will need to be specified as an integer followed by the duration unit, without a space." +echo "For example, to purge all data, but retain the past 3 months, specify 3mo for the duration." +echo "The duration units are as follows:" +echo " mo - month(s)" +echo " w - week(s)" +echo " d - day(s)" + +while true; do +echo "" +read -p 'Enter the duration of past data that you would like to retain: ' duration +duration=$(echo $duration | tr '[:upper:]' '[:lower:]') + +originalIFS=$IFS + +case "${duration}" in + + [0-9]+mo$) + break + ;; + + [0-9]+w$) + break + ;; + + [0-9]+d$) + break + ;; + + *) + echo "" + echo "Invalid duration." + ;; + +esac +done + +IFS=$originalIFS + +echo "Cleaning InfluxDb and saving only the past ${duration}" +docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - ${duration}\"" \ No newline at end of file From e53280447415a81ab0063c1277c467798dece9ef Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 13:42:21 -0400 Subject: [PATCH 60/76] move to proper dir --- salt/common/tools/{ => sbin}/so-influxdb-clear | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) rename salt/common/tools/{ => sbin}/so-influxdb-clear (59%) diff --git a/salt/common/tools/so-influxdb-clear b/salt/common/tools/sbin/so-influxdb-clear similarity index 59% rename from salt/common/tools/so-influxdb-clear rename to salt/common/tools/sbin/so-influxdb-clear index 38ef6491f..c6d5446e9 100644 --- a/salt/common/tools/so-influxdb-clear +++ b/salt/common/tools/sbin/so-influxdb-clear @@ -1,3 +1,19 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . echo "This script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." echo "The duration will need to be specified as an integer followed by the duration unit, without a space." From a5ee8fb59d8a37ba71e1a596d20e78b103c97dfe Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 14:56:53 -0400 Subject: [PATCH 61/76] fix the issues with so-influxdb-clear --- salt/common/tools/sbin/so-influxdb-clear | 44 ++++++++++-------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-clear b/salt/common/tools/sbin/so-influxdb-clear index c6d5446e9..809b17381 100644 --- a/salt/common/tools/sbin/so-influxdb-clear +++ b/salt/common/tools/sbin/so-influxdb-clear @@ -15,8 +15,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +modurregex="^[0-9]+mo$" +wdurregex="^[0-9]+w$" +ddurregex="^[0-9]+d$" + +echo "" echo "This script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." echo "The duration will need to be specified as an integer followed by the duration unit, without a space." +echo "" echo "For example, to purge all data, but retain the past 3 months, specify 3mo for the duration." echo "The duration units are as follows:" echo " mo - month(s)" @@ -24,35 +30,21 @@ echo " w - week(s)" echo " d - day(s)" while true; do -echo "" -read -p 'Enter the duration of past data that you would like to retain: ' duration -duration=$(echo $duration | tr '[:upper:]' '[:lower:]') + echo "" + read -p 'Enter the duration of past data that you would like to retain: ' duration + duration=$(echo $duration | tr '[:upper:]' '[:lower:]') -originalIFS=$IFS - -case "${duration}" in - - [0-9]+mo$) + if [[ "$duration" =~ $modurregex ]] || [[ "$duration" =~ $wdurregex ]] || [[ "$duration" =~ $ddurregex ]]; then break - ;; + fi - [0-9]+w$) - break - ;; - - [0-9]+d$) - break - ;; - - *) - echo "" - echo "Invalid duration." - ;; - -esac + echo "" + echo "Invalid duration." done -IFS=$originalIFS - echo "Cleaning InfluxDb and saving only the past ${duration}" -docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - ${duration}\"" \ No newline at end of file +if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - $duration\""; then + echo "InfluxDb clean complete." +else + echo "Something went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." +fi From 44de6110972dd9d857e11b7b066d04ba206c43a1 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 14:57:39 -0400 Subject: [PATCH 62/76] rename to so-influxdb-clean --- salt/common/tools/sbin/{so-influxdb-clear => so-influxdb-clean} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename salt/common/tools/sbin/{so-influxdb-clear => so-influxdb-clean} (100%) diff --git a/salt/common/tools/sbin/so-influxdb-clear b/salt/common/tools/sbin/so-influxdb-clean similarity index 100% rename from salt/common/tools/sbin/so-influxdb-clear rename to salt/common/tools/sbin/so-influxdb-clean From db98b7ed276cb0df0661798180771a7379562268 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 15:08:11 -0400 Subject: [PATCH 63/76] verify with user before proceedig to clean --- salt/common/tools/sbin/so-influxdb-clean | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-clean b/salt/common/tools/sbin/so-influxdb-clean index 809b17381..453d5b2e4 100644 --- a/salt/common/tools/sbin/so-influxdb-clean +++ b/salt/common/tools/sbin/so-influxdb-clean @@ -19,11 +19,9 @@ modurregex="^[0-9]+mo$" wdurregex="^[0-9]+w$" ddurregex="^[0-9]+d$" -echo "" -echo "This script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." +echo -e "\nThis script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." echo "The duration will need to be specified as an integer followed by the duration unit, without a space." -echo "" -echo "For example, to purge all data, but retain the past 3 months, specify 3mo for the duration." +echo -e "\nFor example, to purge all data, but retain the past 3 months, specify 3mo for the duration." echo "The duration units are as follows:" echo " mo - month(s)" echo " w - week(s)" @@ -38,13 +36,18 @@ while true; do break fi - echo "" - echo "Invalid duration." + echo -e "\nInvalid duration." done -echo "Cleaning InfluxDb and saving only the past ${duration}" -if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - $duration\""; then - echo "InfluxDb clean complete." +echo -e "\nInfluxDB will now be cleaned and leave only the past $duration worth of data." +read -r -p "Are you sure you want to continue? [y/N] " yorn +if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then + echo -e "\nCleaning InfluxDb and saving only the past $duration. This may could take several minutes depending on how much data needs to be cleaned." + if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-01-01T00:00:00.0000000Z' AND \"time\" <= now() - $duration\""; then + echo -e "\nInfluxDb clean complete." + else + echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." + fi else - echo "Something went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." + echo -e "\nExiting as requested." fi From 0ea4c99102bbd3aecc11c164395ece5a8edd5dc8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 15:32:53 -0400 Subject: [PATCH 64/76] remove support for months as it isnt supported in InfluxQL --- salt/common/tools/sbin/so-influxdb-clean | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-clean b/salt/common/tools/sbin/so-influxdb-clean index 453d5b2e4..954abfa8f 100644 --- a/salt/common/tools/sbin/so-influxdb-clean +++ b/salt/common/tools/sbin/so-influxdb-clean @@ -15,15 +15,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -modurregex="^[0-9]+mo$" wdurregex="^[0-9]+w$" ddurregex="^[0-9]+d$" echo -e "\nThis script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." echo "The duration will need to be specified as an integer followed by the duration unit, without a space." -echo -e "\nFor example, to purge all data, but retain the past 3 months, specify 3mo for the duration." +echo -e "\nFor example, to purge all data, but retain the past 12 weeks, specify 12w for the duration." echo "The duration units are as follows:" -echo " mo - month(s)" echo " w - week(s)" echo " d - day(s)" @@ -32,7 +30,7 @@ while true; do read -p 'Enter the duration of past data that you would like to retain: ' duration duration=$(echo $duration | tr '[:upper:]' '[:lower:]') - if [[ "$duration" =~ $modurregex ]] || [[ "$duration" =~ $wdurregex ]] || [[ "$duration" =~ $ddurregex ]]; then + if [[ "$duration" =~ $wdurregex ]] || [[ "$duration" =~ $ddurregex ]]; then break fi From 925be17d51be12a6a49caec3f3c5f3d194fa04ad Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 15:59:18 -0400 Subject: [PATCH 65/76] clean some commas in so-influxdb-clean --- salt/common/tools/sbin/so-influxdb-clean | 4 ++-- salt/common/tools/sbin/so-influxdb-migrate | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 salt/common/tools/sbin/so-influxdb-migrate diff --git a/salt/common/tools/sbin/so-influxdb-clean b/salt/common/tools/sbin/so-influxdb-clean index 954abfa8f..635ac6a02 100644 --- a/salt/common/tools/sbin/so-influxdb-clean +++ b/salt/common/tools/sbin/so-influxdb-clean @@ -19,8 +19,8 @@ wdurregex="^[0-9]+w$" ddurregex="^[0-9]+d$" echo -e "\nThis script is used to reduce the size of InfluxDB by removing old data and retaining only the duration specified." -echo "The duration will need to be specified as an integer followed by the duration unit, without a space." -echo -e "\nFor example, to purge all data, but retain the past 12 weeks, specify 12w for the duration." +echo "The duration will need to be specified as an integer followed by the duration unit without a space." +echo -e "\nFor example, to purge all data but retain the past 12 weeks, specify 12w for the duration." echo "The duration units are as follows:" echo " w - week(s)" echo " d - day(s)" diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate new file mode 100644 index 000000000..c27997bd5 --- /dev/null +++ b/salt/common/tools/sbin/so-influxdb-migrate @@ -0,0 +1,16 @@ +#!/bin/bash + +# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . From 20188549f731ca133d4fc69402f3ea31bc4e0757 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 5 May 2021 19:28:16 -0400 Subject: [PATCH 66/76] add the logic for so-influxdb-migrate --- salt/common/tools/sbin/so-influxdb-migrate | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate index c27997bd5..3b21d099d 100644 --- a/salt/common/tools/sbin/so-influxdb-migrate +++ b/salt/common/tools/sbin/so-influxdb-migrate @@ -14,3 +14,33 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . + +echo -e "\nThis script is used to reduce the size of InfluxDB by downsampling old data into the so_long_term retention policy." + +echo -e "\nInfluxDB will now be migrated. This could take a few hours depending on how large the database is and hardware resources available." +read -r -p "Are you sure you want to continue? [y/N] " yorn +if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then + echo -e "\nMigrating InfluxDb started at `date`. This may take several hours depending on how much data needs to be moved." + day=0 + startdate=`date` + + while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do + # why 2020-07-21? + echo "Migration of 2020-07-21T00:00:00.0000000Z' + ${day}d started at $startdate and completed at `date`." + + echo "Started cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`" + if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d\""; then + echo -e "Completed cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`\n" + else + echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." + fi + + ((day=day+1)) + startdate=`date` + done + + echo -e "\nInfluxDb data migration complete." + +else + echo -e "\nExiting as requested." +fi From 9dbb9f519b40765972e86bc1993796d268a2c5d2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 09:14:49 -0400 Subject: [PATCH 67/76] create so_short_term rp as default so that autogen can just be dropped once data is downsampled --- salt/common/tools/sbin/so-influxdb-migrate | 14 +++++++------- salt/influxdb/defaults.yaml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate index 3b21d099d..3411dc39f 100644 --- a/salt/common/tools/sbin/so-influxdb-migrate +++ b/salt/common/tools/sbin/so-influxdb-migrate @@ -24,16 +24,16 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then day=0 startdate=`date` - while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do + while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM \"autogen\"./.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do # why 2020-07-21? echo "Migration of 2020-07-21T00:00:00.0000000Z' + ${day}d started at $startdate and completed at `date`." - echo "Started cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`" - if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d\""; then - echo -e "Completed cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`\n" - else - echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." - fi +# echo "Started cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`" +# if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d\""; then +# echo -e "Completed cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`\n" +# else +# echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." +# fi ((day=day+1)) startdate=`date` diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 0c9a09080..7ba83dd6d 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -1,6 +1,6 @@ influxdb: retention_policies: - autogen: + so_short_term: default: True duration: 30d shard_duration: 1d From da528e802f2797cdcc4a6b920b87f3476bd74399 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 12:52:47 -0400 Subject: [PATCH 68/76] ensure migration script doesnt migrate the current days data and fix downsample cq to move from so_short_term rp --- salt/common/tools/sbin/so-influxdb-migrate | 21 +++++++++++---------- salt/influxdb/init.sls | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-migrate b/salt/common/tools/sbin/so-influxdb-migrate index 3411dc39f..4d65cba95 100644 --- a/salt/common/tools/sbin/so-influxdb-migrate +++ b/salt/common/tools/sbin/so-influxdb-migrate @@ -26,17 +26,18 @@ if [[ "$yorn" =~ ^([yY][eE][sS]|[yY])$ ]]; then while docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"SELECT mean(*) INTO \"so_long_term\".:MEASUREMENT FROM \"autogen\"./.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d GROUP BY time(5m),*\""; do # why 2020-07-21? - echo "Migration of 2020-07-21T00:00:00.0000000Z' + ${day}d started at $startdate and completed at `date`." + migrationdate=`date -d "2020-07-21 + ${day} days" +"%y-%m-%d"` + echo "Migration of $migrationdate started at $startdate and completed at `date`." -# echo "Started cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`" -# if docker exec -t so-influxdb /bin/bash -c "influx -ssl -unsafeSsl -database telegraf -execute \"DELETE FROM /.*/ WHERE \"time\" >= '2020-07-21T00:00:00.0000000Z' + ${day}d AND \"time\" <= '2020-07-21T00:00:00.0000000Z' + $((day+1))d\""; then -# echo -e "Completed cleaning 2020-07-21T00:00:00.0000000Z' + ${day}d at `date`\n" -# else -# echo -e "\nSomething went wrong with cleaning InfluxDB. Please verify that the so-influxdb Docker container is running, and check the log at /opt/so/log/influxdb/influxdb.log for any details." -# fi - - ((day=day+1)) - startdate=`date` + newdaytomigrate=$(date -d "$migrationdate + 1 days" +"%s") + today=$(date +"%s") + if [ $newdaytomigrate -ge $today ]; then + break + else + ((day=day+1)) + startdate=`date` + echo -e "\nMigrating the next day's worth of data." + fi done echo -e "\nInfluxDb data migration complete." diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index c6120d292..2617578a4 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -117,7 +117,7 @@ so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM /.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* + - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM \"so_short_term\"./.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* - ssl: True - verify_ssl: /etc/pki/ca.crt - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] From 78240b4b5232513e32521bed339157df32940a9c Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 17:49:02 -0400 Subject: [PATCH 69/76] change retry command --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 965b9369c..6cb292aba 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that there is at least 1 minion connected - retry 50 10 "(( $(salt-run manage.up -l error | wc -l) >= 1 ))" + retry 50 10 "salt-run manage.up -l error" $MINIONID echo "" echo "Ensuring python modules for Salt are installed and patched." From b8ad80ae35c0cc352ab3b942efcd4ca82c1329f9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 17:49:40 -0400 Subject: [PATCH 70/76] update comment --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6cb292aba..bf09e2b2e 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -801,7 +801,7 @@ else echo "Starting Salt Master service." systemctl start salt-master - # Testing that that salt-master is up by checking that there is at least 1 minion connected + # Testing that that salt-master is up by checking that is it connected to itself retry 50 10 "salt-run manage.up -l error" $MINIONID echo "" From 6474c296e18240ee028c234f83f69eeb9e23bfaf Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 20:26:13 -0400 Subject: [PATCH 71/76] dont need to specify dest rp --- salt/influxdb/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 2617578a4..c6120d292 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -117,7 +117,7 @@ so_downsample_cq: influxdb_continuous_query.present: - name: so_downsample_cq - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM \"so_short_term\"./.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* + - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM /.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* - ssl: True - verify_ssl: /etc/pki/ca.crt - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] From 5d3b147b42aa1040e11af75dae0fec34494c2e8d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 6 May 2021 20:32:26 -0400 Subject: [PATCH 72/76] change retry command --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index bf09e2b2e..751a2413c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-run manage.up -l error" $MINIONID + retry 50 10 "salt-run manage.up -l error | grep $MINIONID" echo "" echo "Ensuring python modules for Salt are installed and patched." From 46223e0b3055e8c0af6d424ab23df399421d0627 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 May 2021 08:59:47 -0400 Subject: [PATCH 73/76] add quotes around minionid --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 751a2413c..6fb139909 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-run manage.up -l error | grep $MINIONID" + retry 50 10 "salt-run manage.up -l error | grep \"$MINIONID\"" echo "" echo "Ensuring python modules for Salt are installed and patched." From d0e084b8eab150719b63925714a8dbc5fce19927 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 May 2021 10:20:04 -0400 Subject: [PATCH 74/76] change command to test if salt-master is accepting connections --- salt/common/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 6fb139909..77a2b6018 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-run manage.up -l error | grep \"$MINIONID\"" + retry 50 10 "salt-call state.show_top -l error" echo "" echo "Ensuring python modules for Salt are installed and patched." From 009f7617c1c77426c8a20c7ce62b3ec97dec6472 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 May 2021 12:47:22 -0400 Subject: [PATCH 75/76] check salt-master is responding --- salt/common/tools/sbin/soup | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 77a2b6018..51b7bc29c 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -834,6 +834,10 @@ else echo "" echo "Starting Salt Master service." systemctl start salt-master + + # Testing that that salt-master is up by checking that is it connected to itself + retry 50 10 "salt-call state.show_top -l error" + echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True postupgrade_changes From 9eb63b17f9f571282836b48a1343a1f6c265ff6d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Fri, 7 May 2021 14:48:02 -0400 Subject: [PATCH 76/76] exit if retry fails --- salt/common/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/common/tools/sbin/soup b/salt/common/tools/sbin/soup index 51b7bc29c..620737c16 100755 --- a/salt/common/tools/sbin/soup +++ b/salt/common/tools/sbin/soup @@ -802,7 +802,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" + retry 50 10 "salt-call state.show_top -l error" || exit 1 echo "" echo "Ensuring python modules for Salt are installed and patched." @@ -836,7 +836,7 @@ else systemctl start salt-master # Testing that that salt-master is up by checking that is it connected to itself - retry 50 10 "salt-call state.show_top -l error" + retry 50 10 "salt-call state.show_top -l error" || exit 1 echo "Running a highstate. This could take several minutes." salt-call state.highstate -l info queue=True