influx upgrade

salt/common/tools/sbin/so-influxdb-manage

@@ -12,7 +12,6 @@ usage() {
     echo ""
     echo "Supported Operations:"
     echo "  setup          Loads all templates and creates all required buckets"
-    echo "  templateapply  Applies a single template file, requires: <template-path>"
     echo "  userlist       Lists users"
     echo "  useradd        Adds a new user, requires: <email>"
     echo "  userdel        Removes an existing user, requires: <email>"
@@ -108,18 +107,17 @@ change_password() {
   check_response "$response"
 }
 
-apply_template() {
+apply_templates() {
   token=$1
   oid=$2
   stackid=$3
-  file=$4
+  template_objects_array=$4
-  content=$(cat $file)
+  body="{\"orgID\":\"$oid\",\"stackID\":\"$stackid\",\"templates\":$template_objects_array}"
-  body="{\"orgID\":\"$oid\",\"stackID\":\"$stackid\",\"template\":{\"contents\":$content}}"
   response=$(curl -sk https://localhost:8086/api/v2/templates/apply -X POST -d "$body" -H "Authorization: Token $token")
   check_response "$response"
 }
 
-create_bucket() {
+setup_bucket() {
   token=$1
   oid=$2
   name=$3
@@ -127,50 +125,58 @@ create_bucket() {
   shardduration=$5
 
   response=$(curl -sk "https://localhost:8086/api/v2/buckets?orgID=$oid&name=$name" -H "Authorization: Token $token")
-  check_response "$response"
   bucketid=$(echo "$response" | jq -r ".buckets[0].id")
-  if [[ -z "$stackid" || "$stackid" == null ]]; then
+  if [[ -z "$bucketid" || "$bucketid" == null ]]; then
-    response=$(curl -sk https://localhost:8086/api/v2/buckets -X POST -d "{\"name\":\"$name\",\"orgID\":\"oid\"}" -H "Authorization: Token $token")
+    response=$(curl -sk https://localhost:8086/api/v2/buckets -X POST -d "{\"name\":\"$name\",\"orgID\":\"$oid\"}" -H "Authorization: Token $token")
     check_response "$response"
     bucketid=$(echo "$response" | jq -r .id)
   fi
-  response=$(curl -sk "https://localhost:8086/api/v2/buckets/$bucketid" -d "{\"name\":\"$name\",\"retentionRules\":[{\"everySeconds\":$age,\"shardGroupDurationSeconds\":$shardduration,\"type\":\"expire\"}]}" -H "Authorization: Token $token")
+  response=$(curl -sk "https://localhost:8086/api/v2/buckets/$bucketid" -X PATCH -d "{\"name\":\"$name\",\"retentionRules\":[{\"everySeconds\":$age,\"shardGroupDurationSeconds\":$shardduration,\"type\":\"expire\"}]}" -H "Authorization: Token $token")
   check_response "$response"
 }
 
 case "$OP" in
 
-  templateload)
-    [ $# -ne 1 ] && usage
-    file=$1
-    log "Applying template file; file=$file"
-    token=$(lookup_pillar_secret influx_token)
-    oid=$(lookup_org_id "$token")
-    stackid=$(lookup_stack_id "$token" "$oid")
-    apply_template "$token" "$oid" "$stackid" "$file"
-    ;;
-
   setup)
     log "Ensuring organization is setup correctly"
-    token=$(lookup_pillar_secret influx_token)
-    oid=$(lookup_org_id "$token")
 
-    # Load templates
+    # Load templates if at least one has been modified since the last setup
-    stackid=$(lookup_stack_id "$token" "$oid")
+    newest=$(ls -1t /opt/so/conf/influxdb/templates/ | head -1)
-    for file in /opt/so/conf/influxdb/templates/*; do
+    if [ /opt/so/conf/influxdb/templates/$newest -nt /opt/so/conf/influxdb/last_template_setup ]; then
-      log "Ensuring template is loaded; template=$file"
+      log "Updating templates"
-      apply_template "$token" "$oid" "$stackid" "$file"
+      token=$(lookup_pillar_secret influx_token)
-    done
+      oid=$(lookup_org_id "$token")
+      stackid=$(lookup_stack_id "$token" "$oid")
+      for file in /opt/so/conf/influxdb/templates/*; do
+        if [[ "$templates_array" != "" ]]; then
+          templates_array="$templates_array,"
+        fi
+        template=$(cat "$file")
+        templates_array="$templates_array{\"contents\":$template}"
+      done
+      apply_templates "$token" "$oid" "$stackid" "[$templates_array]"
+      echo $(date) > /opt/so/conf/influxdb/last_template_setup
+    else
+      log "Templates have not been modified since last setup"
+    fi
 
-    # Setup buckets and retention periods
+    # Setup buckets and retention periods if at least one has been modified since the last setup
-    for rp in so_short_term so_long_term; do
+    if [ /opt/so/conf/influxdb/buckets.json -nt /opt/so/conf/influxdb/last_bucket_setup ]; then
-      bucket=telegraf/$rp
+      log "Updating buckets and retention periods"
-      log "Ensuring bucket is created and configured; bucket=$bucket"
+      token=$(lookup_pillar_secret influx_token)
-      age=$(cat /opt/so/conf/influxdb/buckets.json | jq -r .$rp.duration)
+      oid=$(lookup_org_id "$token")
-      shard_duration=$(cat /opt/so/conf/influxdb/buckets.json | jq -r .$rp.shard_duration)
+      for rp in so_short_term so_long_term; do
-      create_bucket "$token" "$oid" "$bucket" "$age" "$shard_duration"
+        bucket=telegraf/$rp
-    done
+        log "Ensuring bucket is created and configured; bucket=$bucket"
-    ;;    
+        age=$(cat /opt/so/conf/influxdb/buckets.json | jq -r .$rp.duration)
+        shard_duration=$(cat /opt/so/conf/influxdb/buckets.json | jq -r .$rp.shard_duration)
+        setup_bucket "$token" "$oid" "$bucket" "$age" "$shard_duration"
+      done
+      echo $(date) > /opt/so/conf/influxdb/last_bucket_setup
+    else
+      log "Buckets have not been modified since last setup"
+    fi
+    ;;
 
   userlist)
     log "Listing existing users"

salt/influxdb/init.sls

@@ -57,7 +57,7 @@ influxdb-templates:
 
 influxdb-setup:
   cmd.run:
-    - name: /usr/sbin/so-influxdb-setup
+    - name: /usr/sbin/so-influxdb-manage setup &>> /opt/so/log/influxdb/setup.log
 
 so-influxdb:
   docker_container.running:

salt/influxdb/templates/container.json

@@ -1 +1 @@
-[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-container"},"spec":{"language":"flux","name":"container","query":"import \"array\"\n\ndynamic = from(bucket: \"telegraf\")\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")\n  |> filter(fn: (r) => r[\"host\"] == v.host)\n  |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n  |> keep(columns: [\"container_name\"])\n  |> rename(fn: (column) => \"_value\")\n  |> unique()\n\nstatic = array.from(\n    rows: [\n        {\n            _value: \"All\",\n        },\n    ],\n)\n\nunion(tables: [static, dynamic])","selected":["cool_gauss"],"type":"query"}}]
+[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-container"},"spec":{"language":"flux","name":"container","query":"import \"array\"\n\ndynamic = from(bucket: \"telegraf/so_short_term\")\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")\n  |> filter(fn: (r) => r[\"host\"] == v.host)\n  |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n  |> keep(columns: [\"container_name\"])\n  |> rename(fn: (column) => \"_value\")\n  |> unique()\n\nstatic = array.from(\n    rows: [\n        {\n            _value: \"All\",\n        },\n    ],\n)\n\nunion(tables: [static, dynamic])","selected":["cool_gauss"],"type":"query"}}]

salt/influxdb/templates/host.json

@@ -1 +1 @@
-[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-host"},"spec":{"language":"flux","name":"host","query":"import \"influxdata/influxdb/schema\"\nimport \"array\"\n\ndynamic = schema.tagValues(bucket: \"telegraf\", tag: \"host\")\n\nstatic = array.from(\n    rows: [\n        {\n            _value: \"All\",\n        },\n    ],\n)\n\nunion(tables: [static, dynamic])","selected":["dev"],"type":"query"}}]
+[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-host"},"spec":{"language":"flux","name":"host","query":"import \"influxdata/influxdb/schema\"\nimport \"array\"\n\ndynamic = schema.tagValues(bucket: \"telegraf/so_short_term\", tag: \"host\")\n\nstatic = array.from(\n    rows: [\n        {\n            _value: \"All\",\n        },\n    ],\n)\n\nunion(tables: [static, dynamic])","selected":["dev"],"type":"query"}}]

salt/influxdb/templates/role.json

@@ -1 +1 @@
-[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-role"},"spec":{"language":"flux","name":"role","query":"import \"influxdata/influxdb/schema\"\nimport \"array\"\n\ndynamic = schema.tagValues(bucket: \"telegraf\", tag: \"role\")\n\nstatic = array.from(\n    rows: [\n        {\n            _value: \"All\",\n        },\n    ],\n)\n\nunion(tables: [static, dynamic])","selected":["standalone"],"type":"query"}}]
+[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-role"},"spec":{"language":"flux","name":"role","query":"import \"influxdata/influxdb/schema\"\nimport \"array\"\n\ndynamic = schema.tagValues(bucket: \"telegraf/so_short_term\", tag: \"role\")\n\nstatic = array.from(\n    rows: [\n        {\n            _value: \"All\",\n        },\n    ],\n)\n\nunion(tables: [static, dynamic])","selected":["standalone"],"type":"query"}}]