From 022f9ea76ed67303caefbfc26977000cb7949440 Mon Sep 17 00:00:00 2001 From: William Wernert Date: Thu, 24 Jun 2021 10:45:12 -0400 Subject: [PATCH 1/6] Add Elasticsearch and Kibana to list of services that use webuser creds --- setup/so-whiptail | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/so-whiptail b/setup/so-whiptail index 031265065..afd691632 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -220,7 +220,7 @@ whiptail_create_web_user() { [ -n "$TESTING" ] && return WEBUSER=$(whiptail --title "$whiptail_title" --inputbox \ - "Please enter an email address to create an administrator account for the web interface.\n\nThis will also be used for TheHive, Cortex, and Fleet." 12 60 "$1" 3>&1 1>&2 2>&3) + "Please enter an email address to create an administrator account for the web interface.\n\nThis will also be used for Elasticsearch, Kibana, TheHive, Cortex, and Fleet." 12 60 "$1" 3>&1 1>&2 2>&3) local exitstatus=$? whiptail_check_exitstatus $exitstatus From 21c9388ee63e46dee1102b9190c47b7bfbb9389b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 28 Jun 2021 16:12:36 -0400 Subject: [PATCH 2/6] generate measurement list and cq for each --- salt/influxdb/defaults.yaml | 6 +++++- salt/influxdb/init.sls | 9 +++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 7ba83dd6d..1c7faf022 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -10,4 +10,8 @@ influxdb: shard_duration: 7d downsample: so_long_term: - resolution: 5m \ No newline at end of file + resolution: 5m + measurements: +{% for measurement in salt['cmd.run']('docker exec -t so-influxdb /bin/bash -c "influx -format json -ssl -unsafeSsl -database telegraf -execute \"show measurements\"" | jq -r ".results[0].series[0].values[][0]"') %} + - {{ measurement }} +{% endfor %} diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 346d971fd..fb84f8bc8 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -113,11 +113,12 @@ telegraf_database: {% endfor %} {% for dest_rp in influxdb.downsample.keys() %} -so_downsample_cq: + {% for measurement in influxdb.downsample[dest_rp].measurements %} +so_downsample_{{measurement}}_cq: influxdb_continuous_query.present: - - name: so_downsample_cq + - name: so_downsample_{{measurement}}_cq - database: telegraf - - query: SELECT mean(*) INTO "{{dest_rp}}".:MEASUREMENT FROM /.*/ GROUP BY time({{influxdb.downsample[dest_rp].resolution}}),* + - query: SELECT mean(*) INTO "{{dest_rp}}"."{{measurement}}" FROM "{{measurement}}" GROUP BY time({{influxdb.downsample[dest_rp].resolution}}) - ssl: True - verify_ssl: /etc/pki/ca.crt - cert: ['/etc/pki/influxdb.crt', '/etc/pki/influxdb.key'] @@ -126,7 +127,7 @@ so_downsample_cq: - docker_container: so-influxdb - influxdb_database: telegraf_database - file: influxdb_continuous_query.present_patch - - sls: salt.python3-influxdb + {% endfor %} {% endfor %} {% endif %} From 1c516daa96cbd9e7cbe6f88f062591b103b655c8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 28 Jun 2021 17:05:32 -0400 Subject: [PATCH 3/6] fix measurement list --- salt/influxdb/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 1c7faf022..651722315 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -12,6 +12,6 @@ influxdb: so_long_term: resolution: 5m measurements: -{% for measurement in salt['cmd.run']('docker exec -t so-influxdb /bin/bash -c "influx -format json -ssl -unsafeSsl -database telegraf -execute \"show measurements\"" | jq -r ".results[0].series[0].values[][0]"') %} +{% for measurement in salt['cmd.shell']('docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -database telegraf -execute "show measurements" | jq -r .results[0].series[0].values[][0]').splitlines() %} - {{ measurement }} {% endfor %} From 6b68a39cbe87384628172a2e5bb6b04401a2c6bf Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 07:46:25 -0400 Subject: [PATCH 4/6] handle senario where there are no measurements --- salt/influxdb/defaults.yaml | 8 ++++++-- salt/influxdb/init.sls | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 651722315..bfcee7927 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -1,3 +1,5 @@ +{% set measurements = salt['cmd.shell']('docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -database telegraf -execute "show measurements" | jq -r .results[0].series[0].values[][0]') %} + influxdb: retention_policies: so_short_term: @@ -11,7 +13,9 @@ influxdb: downsample: so_long_term: resolution: 5m +{% if measurements is defined %} measurements: -{% for measurement in salt['cmd.shell']('docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -database telegraf -execute "show measurements" | jq -r .results[0].series[0].values[][0]').splitlines() %} + {% for measurement in measurements.splitlines() %} - {{ measurement }} -{% endfor %} + {% endfor %} +{% endif %} diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index fb84f8bc8..37ebe39a5 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -113,7 +113,7 @@ telegraf_database: {% endfor %} {% for dest_rp in influxdb.downsample.keys() %} - {% for measurement in influxdb.downsample[dest_rp].measurements %} + {% for measurement in influxdb.downsample[dest_rp].get('measurements', []) %} so_downsample_{{measurement}}_cq: influxdb_continuous_query.present: - name: so_downsample_{{measurement}}_cq From bf8bba7b84659463aece0b668db6256382c8b09f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 08:57:51 -0400 Subject: [PATCH 5/6] only set measurements if conditions are met --- salt/influxdb/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index bfcee7927..dae7f1cb4 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -13,7 +13,7 @@ influxdb: downsample: so_long_term: resolution: 5m -{% if measurements is defined %} +{% if 'jq: error' not in measurements and 'Error response from daemon' not in measurements and 'parse error:' not in measurements and measurements|length > 0 %} measurements: {% for measurement in measurements.splitlines() %} - {{ measurement }} From ca152ab04cddf3b3649d121346f2c63f27da1733 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 29 Jun 2021 09:54:17 -0400 Subject: [PATCH 6/6] redefine measurements --- salt/influxdb/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index dae7f1cb4..205c2ba67 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -1,4 +1,4 @@ -{% set measurements = salt['cmd.shell']('docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -database telegraf -execute "show measurements" | jq -r .results[0].series[0].values[][0]') %} +{% set measurements = salt['cmd.shell']('docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -database telegraf -execute "show measurements" 2> /root/measurement_query.log | jq -r .results[0].series[0].values[]?[0] 2>> /root/measurement_query.log') %} influxdb: retention_policies: @@ -13,7 +13,7 @@ influxdb: downsample: so_long_term: resolution: 5m -{% if 'jq: error' not in measurements and 'Error response from daemon' not in measurements and 'parse error:' not in measurements and measurements|length > 0 %} +{% if measurements|length > 0 %} measurements: {% for measurement in measurements.splitlines() %} - {{ measurement }}