From 7dee2686ac3eec272f1074cc0133b575d37452df Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 10 Feb 2023 18:19:31 -0500 Subject: [PATCH] influx upgrade --- salt/common/tools/sbin/so-influxdb-manage | 80 ++++++++++++----------- salt/influxdb/init.sls | 2 +- salt/influxdb/templates/container.json | 2 +- salt/influxdb/templates/host.json | 2 +- salt/influxdb/templates/role.json | 2 +- 5 files changed, 47 insertions(+), 41 deletions(-) diff --git a/salt/common/tools/sbin/so-influxdb-manage b/salt/common/tools/sbin/so-influxdb-manage index e5f480369..40eb2e1e2 100644 --- a/salt/common/tools/sbin/so-influxdb-manage +++ b/salt/common/tools/sbin/so-influxdb-manage @@ -12,7 +12,6 @@ usage() { echo "" echo "Supported Operations:" echo " setup Loads all templates and creates all required buckets" - echo " templateapply Applies a single template file, requires: " echo " userlist Lists users" echo " useradd Adds a new user, requires: " echo " userdel Removes an existing user, requires: " @@ -108,18 +107,17 @@ change_password() { check_response "$response" } -apply_template() { +apply_templates() { token=$1 oid=$2 stackid=$3 - file=$4 - content=$(cat $file) - body="{\"orgID\":\"$oid\",\"stackID\":\"$stackid\",\"template\":{\"contents\":$content}}" + template_objects_array=$4 + body="{\"orgID\":\"$oid\",\"stackID\":\"$stackid\",\"templates\":$template_objects_array}" response=$(curl -sk https://localhost:8086/api/v2/templates/apply -X POST -d "$body" -H "Authorization: Token $token") check_response "$response" } -create_bucket() { +setup_bucket() { token=$1 oid=$2 name=$3 @@ -127,50 +125,58 @@ create_bucket() { shardduration=$5 response=$(curl -sk "https://localhost:8086/api/v2/buckets?orgID=$oid&name=$name" -H "Authorization: Token $token") - check_response "$response" bucketid=$(echo "$response" | jq -r ".buckets[0].id") - if [[ -z "$stackid" || "$stackid" == null ]]; then - response=$(curl -sk https://localhost:8086/api/v2/buckets -X POST -d "{\"name\":\"$name\",\"orgID\":\"oid\"}" -H "Authorization: Token $token") + if [[ -z "$bucketid" || "$bucketid" == null ]]; then + response=$(curl -sk https://localhost:8086/api/v2/buckets -X POST -d "{\"name\":\"$name\",\"orgID\":\"$oid\"}" -H "Authorization: Token $token") check_response "$response" bucketid=$(echo "$response" | jq -r .id) fi - response=$(curl -sk "https://localhost:8086/api/v2/buckets/$bucketid" -d "{\"name\":\"$name\",\"retentionRules\":[{\"everySeconds\":$age,\"shardGroupDurationSeconds\":$shardduration,\"type\":\"expire\"}]}" -H "Authorization: Token $token") + response=$(curl -sk "https://localhost:8086/api/v2/buckets/$bucketid" -X PATCH -d "{\"name\":\"$name\",\"retentionRules\":[{\"everySeconds\":$age,\"shardGroupDurationSeconds\":$shardduration,\"type\":\"expire\"}]}" -H "Authorization: Token $token") check_response "$response" } case "$OP" in - templateload) - [ $# -ne 1 ] && usage - file=$1 - log "Applying template file; file=$file" - token=$(lookup_pillar_secret influx_token) - oid=$(lookup_org_id "$token") - stackid=$(lookup_stack_id "$token" "$oid") - apply_template "$token" "$oid" "$stackid" "$file" - ;; - setup) log "Ensuring organization is setup correctly" - token=$(lookup_pillar_secret influx_token) - oid=$(lookup_org_id "$token") - # Load templates - stackid=$(lookup_stack_id "$token" "$oid") - for file in /opt/so/conf/influxdb/templates/*; do - log "Ensuring template is loaded; template=$file" - apply_template "$token" "$oid" "$stackid" "$file" - done + # Load templates if at least one has been modified since the last setup + newest=$(ls -1t /opt/so/conf/influxdb/templates/ | head -1) + if [ /opt/so/conf/influxdb/templates/$newest -nt /opt/so/conf/influxdb/last_template_setup ]; then + log "Updating templates" + token=$(lookup_pillar_secret influx_token) + oid=$(lookup_org_id "$token") + stackid=$(lookup_stack_id "$token" "$oid") + for file in /opt/so/conf/influxdb/templates/*; do + if [[ "$templates_array" != "" ]]; then + templates_array="$templates_array," + fi + template=$(cat "$file") + templates_array="$templates_array{\"contents\":$template}" + done + apply_templates "$token" "$oid" "$stackid" "[$templates_array]" + echo $(date) > /opt/so/conf/influxdb/last_template_setup + else + log "Templates have not been modified since last setup" + fi - # Setup buckets and retention periods - for rp in so_short_term so_long_term; do - bucket=telegraf/$rp - log "Ensuring bucket is created and configured; bucket=$bucket" - age=$(cat /opt/so/conf/influxdb/buckets.json | jq -r .$rp.duration) - shard_duration=$(cat /opt/so/conf/influxdb/buckets.json | jq -r .$rp.shard_duration) - create_bucket "$token" "$oid" "$bucket" "$age" "$shard_duration" - done - ;; + # Setup buckets and retention periods if at least one has been modified since the last setup + if [ /opt/so/conf/influxdb/buckets.json -nt /opt/so/conf/influxdb/last_bucket_setup ]; then + log "Updating buckets and retention periods" + token=$(lookup_pillar_secret influx_token) + oid=$(lookup_org_id "$token") + for rp in so_short_term so_long_term; do + bucket=telegraf/$rp + log "Ensuring bucket is created and configured; bucket=$bucket" + age=$(cat /opt/so/conf/influxdb/buckets.json | jq -r .$rp.duration) + shard_duration=$(cat /opt/so/conf/influxdb/buckets.json | jq -r .$rp.shard_duration) + setup_bucket "$token" "$oid" "$bucket" "$age" "$shard_duration" + done + echo $(date) > /opt/so/conf/influxdb/last_bucket_setup + else + log "Buckets have not been modified since last setup" + fi + ;; userlist) log "Listing existing users" diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index 96b1e7b76..efd46ac7d 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -57,7 +57,7 @@ influxdb-templates: influxdb-setup: cmd.run: - - name: /usr/sbin/so-influxdb-setup + - name: /usr/sbin/so-influxdb-manage setup &>> /opt/so/log/influxdb/setup.log so-influxdb: docker_container.running: diff --git a/salt/influxdb/templates/container.json b/salt/influxdb/templates/container.json index 77ff20bde..48f8ac95d 100644 --- a/salt/influxdb/templates/container.json +++ b/salt/influxdb/templates/container.json @@ -1 +1 @@ -[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-container"},"spec":{"language":"flux","name":"container","query":"import \"array\"\n\ndynamic = from(bucket: \"telegraf\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")\n |> filter(fn: (r) => r[\"host\"] == v.host)\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> keep(columns: [\"container_name\"])\n |> rename(fn: (column) => \"_value\")\n |> unique()\n\nstatic = array.from(\n rows: [\n {\n _value: \"All\",\n },\n ],\n)\n\nunion(tables: [static, dynamic])","selected":["cool_gauss"],"type":"query"}}] +[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-container"},"spec":{"language":"flux","name":"container","query":"import \"array\"\n\ndynamic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")\n |> filter(fn: (r) => r[\"host\"] == v.host)\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> keep(columns: [\"container_name\"])\n |> rename(fn: (column) => \"_value\")\n |> unique()\n\nstatic = array.from(\n rows: [\n {\n _value: \"All\",\n },\n ],\n)\n\nunion(tables: [static, dynamic])","selected":["cool_gauss"],"type":"query"}}] diff --git a/salt/influxdb/templates/host.json b/salt/influxdb/templates/host.json index adfdf884f..8d9f663e1 100644 --- a/salt/influxdb/templates/host.json +++ b/salt/influxdb/templates/host.json @@ -1 +1 @@ -[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-host"},"spec":{"language":"flux","name":"host","query":"import \"influxdata/influxdb/schema\"\nimport \"array\"\n\ndynamic = schema.tagValues(bucket: \"telegraf\", tag: \"host\")\n\nstatic = array.from(\n rows: [\n {\n _value: \"All\",\n },\n ],\n)\n\nunion(tables: [static, dynamic])","selected":["dev"],"type":"query"}}] +[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-host"},"spec":{"language":"flux","name":"host","query":"import \"influxdata/influxdb/schema\"\nimport \"array\"\n\ndynamic = schema.tagValues(bucket: \"telegraf/so_short_term\", tag: \"host\")\n\nstatic = array.from(\n rows: [\n {\n _value: \"All\",\n },\n ],\n)\n\nunion(tables: [static, dynamic])","selected":["dev"],"type":"query"}}] diff --git a/salt/influxdb/templates/role.json b/salt/influxdb/templates/role.json index 72b48bdd7..2a895563c 100644 --- a/salt/influxdb/templates/role.json +++ b/salt/influxdb/templates/role.json @@ -1 +1 @@ -[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-role"},"spec":{"language":"flux","name":"role","query":"import \"influxdata/influxdb/schema\"\nimport \"array\"\n\ndynamic = schema.tagValues(bucket: \"telegraf\", tag: \"role\")\n\nstatic = array.from(\n rows: [\n {\n _value: \"All\",\n },\n ],\n)\n\nunion(tables: [static, dynamic])","selected":["standalone"],"type":"query"}}] +[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-role"},"spec":{"language":"flux","name":"role","query":"import \"influxdata/influxdb/schema\"\nimport \"array\"\n\ndynamic = schema.tagValues(bucket: \"telegraf/so_short_term\", tag: \"role\")\n\nstatic = array.from(\n rows: [\n {\n _value: \"All\",\n },\n ],\n)\n\nunion(tables: [static, dynamic])","selected":["standalone"],"type":"query"}}]