From e5c26032c428680f814d91f08191af21c7e7bd17 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Fri, 10 Feb 2023 19:37:59 -0500 Subject: [PATCH] influx upgrade --- salt/common/tools/sbin/so-influxdb-manage | 104 ++++++++++------------ salt/influxdb/curl.config.jinja | 1 + salt/influxdb/init.sls | 8 ++ salt/influxdb/templates/container.json | 2 +- salt/influxdb/templates/host.json | 2 +- salt/influxdb/templates/role.json | 2 +- 6 files changed, 61 insertions(+), 58 deletions(-) create mode 100644 salt/influxdb/curl.config.jinja diff --git a/salt/common/tools/sbin/so-influxdb-manage b/salt/common/tools/sbin/so-influxdb-manage index 40eb2e1e2..964a1b517 100644 --- a/salt/common/tools/sbin/so-influxdb-manage +++ b/salt/common/tools/sbin/so-influxdb-manage @@ -44,10 +44,14 @@ check_response() { fi } +request() { + curl -skK /opt/so/conf/influxdb/curl.config "https://localhost:8086/api/v2/$@" +} + lookup_user_id() { - token=$1 - email=$2 - response=$(curl -sk https://localhost:8086/api/v2/users?limit=100 -H "Authorization: Token $token") + email=$1 + + response=$(request users?limit=100) check_response "$response" uid=$(echo "$response" | jq -r ".users[] | select(.name == \"$email\").id") if [[ -z "$uid" ]]; then @@ -58,8 +62,7 @@ lookup_user_id() { } lookup_org_id() { - token=$1 - response=$(curl -sk https://localhost:8086/api/v2/orgs?limit=100 -H "Authorization: Token $token") + response=$(request orgs?limit=100) check_response "$response" oid=$(echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id") if [[ -z "$oid" ]]; then @@ -70,13 +73,13 @@ lookup_org_id() { } lookup_stack_id() { - token=$1 - oid=$2 - response=$(curl -sk "https://localhost:8086/api/v2/stacks?orgID=$oid&name=Security+Onion" -H "Authorization: Token $token") + oid=$1 + + response=$(request "stacks?orgID=$oid&name=Security+Onion") check_response "$response" stackid=$(echo "$response" | jq -r ".stacks[0].id") if [[ -z "$stackid" || "$stackid" == null ]]; then - response=$(curl -sk https://localhost:8086/api/v2/stacks -X POST -d "{\"name\":\"Security Onion\",\"orgID\":\"$oid\"}" -H "Authorization: Token $token") + response=$(request stacks -X POST -d "{\"name\":\"Security Onion\",\"orgID\":\"$oid\"}") check_response "$response" stackid=$(echo "$response" | jq -r .id) fi @@ -84,17 +87,17 @@ lookup_stack_id() { } add_user_to_org() { - token=$1 - uid=$2 - oid=$3 + uid=$1 + oid=$2 + log "Adding new user to organization" - response=$(curl -sk https://localhost:8086/api/v2/orgs/$oid/members -X POST -d "{\"id\":\"$uid\"}" -H "Authorization: Token $token") + response=$(request orgs/$oid/members -X POST -d "{\"id\":\"$uid\"}") check_response "$response" } change_password() { - token=$1 - uid=$2 + uid=$1 + set +e test -t 0 if [[ $? == 0 ]]; then @@ -103,35 +106,34 @@ change_password() { set -e read -rs pass check_password_and_exit "$pass" - response=$(curl -sk https://localhost:8086/api/v2/users/$uid/password -X POST -d "{\"password\":\"$pass\"}" -H "Authorization: Token $token") + response=$(request users/$uid/password -X POST -d "{\"password\":\"$pass\"}") check_response "$response" } apply_templates() { - token=$1 - oid=$2 - stackid=$3 - template_objects_array=$4 + oid=$1 + stackid=$2 + template_objects_array=$3 + body="{\"orgID\":\"$oid\",\"stackID\":\"$stackid\",\"templates\":$template_objects_array}" - response=$(curl -sk https://localhost:8086/api/v2/templates/apply -X POST -d "$body" -H "Authorization: Token $token") + response=$(request templates/apply -X POST -d "$body") check_response "$response" } setup_bucket() { - token=$1 - oid=$2 - name=$3 - age=$4 - shardduration=$5 + oid=$1 + name=$2 + age=$3 + shardduration=$4 - response=$(curl -sk "https://localhost:8086/api/v2/buckets?orgID=$oid&name=$name" -H "Authorization: Token $token") + response=$(request "buckets?orgID=$oid&name=$name") bucketid=$(echo "$response" | jq -r ".buckets[0].id") if [[ -z "$bucketid" || "$bucketid" == null ]]; then - response=$(curl -sk https://localhost:8086/api/v2/buckets -X POST -d "{\"name\":\"$name\",\"orgID\":\"$oid\"}" -H "Authorization: Token $token") + response=$(request buckets -X POST -d "{\"name\":\"$name\",\"orgID\":\"$oid\"}") check_response "$response" bucketid=$(echo "$response" | jq -r .id) fi - response=$(curl -sk "https://localhost:8086/api/v2/buckets/$bucketid" -X PATCH -d "{\"name\":\"$name\",\"retentionRules\":[{\"everySeconds\":$age,\"shardGroupDurationSeconds\":$shardduration,\"type\":\"expire\"}]}" -H "Authorization: Token $token") + response=$(request buckets/$bucketid -X PATCH -d "{\"name\":\"$name\",\"retentionRules\":[{\"everySeconds\":$age,\"shardGroupDurationSeconds\":$shardduration,\"type\":\"expire\"}]}") check_response "$response" } @@ -144,9 +146,8 @@ case "$OP" in newest=$(ls -1t /opt/so/conf/influxdb/templates/ | head -1) if [ /opt/so/conf/influxdb/templates/$newest -nt /opt/so/conf/influxdb/last_template_setup ]; then log "Updating templates" - token=$(lookup_pillar_secret influx_token) - oid=$(lookup_org_id "$token") - stackid=$(lookup_stack_id "$token" "$oid") + oid=$(lookup_org_id) + stackid=$(lookup_stack_id "$oid") for file in /opt/so/conf/influxdb/templates/*; do if [[ "$templates_array" != "" ]]; then templates_array="$templates_array," @@ -154,7 +155,7 @@ case "$OP" in template=$(cat "$file") templates_array="$templates_array{\"contents\":$template}" done - apply_templates "$token" "$oid" "$stackid" "[$templates_array]" + apply_templates "$oid" "$stackid" "[$templates_array]" echo $(date) > /opt/so/conf/influxdb/last_template_setup else log "Templates have not been modified since last setup" @@ -163,14 +164,13 @@ case "$OP" in # Setup buckets and retention periods if at least one has been modified since the last setup if [ /opt/so/conf/influxdb/buckets.json -nt /opt/so/conf/influxdb/last_bucket_setup ]; then log "Updating buckets and retention periods" - token=$(lookup_pillar_secret influx_token) - oid=$(lookup_org_id "$token") + oid=$(lookup_org_id) for rp in so_short_term so_long_term; do bucket=telegraf/$rp log "Ensuring bucket is created and configured; bucket=$bucket" age=$(cat /opt/so/conf/influxdb/buckets.json | jq -r .$rp.duration) shard_duration=$(cat /opt/so/conf/influxdb/buckets.json | jq -r .$rp.shard_duration) - setup_bucket "$token" "$oid" "$bucket" "$age" "$shard_duration" + setup_bucket "$oid" "$bucket" "$age" "$shard_duration" done echo $(date) > /opt/so/conf/influxdb/last_bucket_setup else @@ -180,8 +180,7 @@ case "$OP" in userlist) log "Listing existing users" - token=$(lookup_pillar_secret influx_token) - response=$(curl -sk https://localhost:8086/api/v2/users -H "Authorization: Token $token") + response=$(request users) check_response "$response" echo "$response" | jq -r '.users[] | "\(.id): \(.name) (\(.status))"' ;; @@ -190,31 +189,28 @@ case "$OP" in [ $# -ne 1 ] && usage email=$1 log "Adding new user; email=$email" - token=$(lookup_pillar_secret influx_token) - oid=$(lookup_org_id "$token") - response=$(curl -sk https://localhost:8086/api/v2/users -X POST -d "{\"name\":\"$email\"}" -H "Authorization: Token $token") + oid=$(lookup_org_id) + response=$(request users -X POST -d "{\"name\":\"$email\"}") check_response "$response" uid=$(echo "$response" | jq -r .id) - add_user_to_org "$token" "$uid" "$oid" - change_password "$token" "$uid" + add_user_to_org "$uid" "$oid" + change_password "$uid" ;; userpass) [ $# -ne 1 ] && usage email=$1 log "Updating user password; email=$email" - token=$(lookup_pillar_secret influx_token) - uid=$(lookup_user_id "$token" "$email") - change_password "$token" "$uid" + uid=$(lookup_user_id "$email") + change_password "$uid" ;; userdel) [ $# -ne 1 ] && usage email=$1 log "Deleting user; email=$email" - token=$(lookup_pillar_secret influx_token) - uid=$(lookup_user_id "$token" "$email") - response=$(curl -sk https://localhost:8086/api/v2/users/$uid -X DELETE -H "Authorization: Token $token") + uid=$(lookup_user_id "$email") + response=$(request users/$uid -X DELETE) check_response "$response" ;; @@ -222,9 +218,8 @@ case "$OP" in [ $# -ne 1 ] && usage email=$1 log "Enabling user; email=$email" - token=$(lookup_pillar_secret influx_token) - uid=$(lookup_user_id "$token" "$email") - response=$(curl -sk https://localhost:8086/api/v2/users/$uid -X PATCH -d "{\"name\":\"$email\",\"status\":\"active\"}" -H "Authorization: Token $token") + uid=$(lookup_user_id "$email") + response=$(request users/$uid -X PATCH -d "{\"name\":\"$email\",\"status\":\"active\"}") check_response "$response" ;; @@ -232,9 +227,8 @@ case "$OP" in [ $# -ne 1 ] && usage email=$1 log "Disabling user; email=$email" - token=$(lookup_pillar_secret influx_token) - uid=$(lookup_user_id "$token" "$email") - response=$(curl -sk https://localhost:8086/api/v2/users/$uid -X PATCH -d "{\"name\":\"$email\",\"status\":\"inactive\"}" -H "Authorization: Token $token") + uid=$(lookup_user_id "$email") + response=$(request users/$uid -X PATCH -d "{\"name\":\"$email\",\"status\":\"inactive\"}") check_response "$response" ;; diff --git a/salt/influxdb/curl.config.jinja b/salt/influxdb/curl.config.jinja new file mode 100644 index 000000000..9f636e851 --- /dev/null +++ b/salt/influxdb/curl.config.jinja @@ -0,0 +1 @@ +header = "Authorization: Token {{ salt['pillar.get']('secrets:influx_token') }}" \ No newline at end of file diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index efd46ac7d..5b77a6b49 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -55,6 +55,14 @@ influxdb-templates: - template: jinja - clean: True +influxdb_curl_config: + file.managed: + - name: /opt/so/conf/influxdb/curl.config + - source: salt://influxdb/curl.config.jinja + - mode: 600 + - show_changes: False + - makedirs: True + influxdb-setup: cmd.run: - name: /usr/sbin/so-influxdb-manage setup &>> /opt/so/log/influxdb/setup.log diff --git a/salt/influxdb/templates/container.json b/salt/influxdb/templates/container.json index 8e3b86530..e97ea246b 100644 --- a/salt/influxdb/templates/container.json +++ b/salt/influxdb/templates/container.json @@ -1 +1 @@ -[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-container"},"spec":{"language":"flux","name":"Container","query":"import \"array\"\n\ndynamic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")\n |> filter(fn: (r) => r[\"host\"] == v.Host)\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> keep(columns: [\"container_name\"])\n |> rename(fn: (column) => \"_value\")\n |> unique()\n\nstatic = array.from(\n rows: [\n {\n _value: \"All\",\n },\n ],\n)\n\nunion(tables: [static, dynamic])","selected":["cool_gauss"],"type":"query"}}] +[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-container"},"spec":{"language":"flux","name":"Container","query":"import \"array\"\n\ndynamic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")\n |> filter(fn: (r) => r[\"host\"] == v.Host)\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> keep(columns: [\"container_name\"])\n |> rename(fn: (column) => \"_value\")\n |> unique()\n\nstatic = array.from(\n rows: [\n {\n _value: \"(All)\",\n },\n ],\n)\n\nunion(tables: [static, dynamic])","selected":["cool_gauss"],"type":"query"}}] diff --git a/salt/influxdb/templates/host.json b/salt/influxdb/templates/host.json index 8e465c803..293884aab 100644 --- a/salt/influxdb/templates/host.json +++ b/salt/influxdb/templates/host.json @@ -1 +1 @@ -[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-host"},"spec":{"language":"flux","name":"Host","query":"import \"influxdata/influxdb/schema\"\nimport \"array\"\n\ndynamic = schema.tagValues(bucket: \"telegraf/so_short_term\", tag: \"host\")\n\nstatic = array.from(\n rows: [\n {\n _value: \"All\",\n },\n ],\n)\n\nunion(tables: [static, dynamic])","selected":["dev"],"type":"query"}}] +[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-host"},"spec":{"language":"flux","name":"Host","query":"import \"influxdata/influxdb/schema\"\nimport \"array\"\n\ndynamic = schema.tagValues(bucket: \"telegraf/so_short_term\", tag: \"host\")\n\nstatic = array.from(\n rows: [\n {\n _value: \"(All)\",\n },\n ],\n)\n\nunion(tables: [static, dynamic])","selected":["dev"],"type":"query"}}] diff --git a/salt/influxdb/templates/role.json b/salt/influxdb/templates/role.json index af65a14f3..b189dddc8 100644 --- a/salt/influxdb/templates/role.json +++ b/salt/influxdb/templates/role.json @@ -1 +1 @@ -[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-role"},"spec":{"language":"flux","name":"Role","query":"import \"influxdata/influxdb/schema\"\nimport \"array\"\n\ndynamic = schema.tagValues(bucket: \"telegraf/so_short_term\", tag: \"role\")\n\nstatic = array.from(\n rows: [\n {\n _value: \"All\",\n },\n ],\n)\n\nunion(tables: [static, dynamic])","selected":["standalone"],"type":"query"}}] +[{"apiVersion":"influxdata.com/v2alpha1","kind":"Variable","metadata":{"name":"variable-role"},"spec":{"language":"flux","name":"Role","query":"import \"influxdata/influxdb/schema\"\nimport \"array\"\n\ndynamic = schema.tagValues(bucket: \"telegraf/so_short_term\", tag: \"role\")\n\nstatic = array.from(\n rows: [\n {\n _value: \"(All)\",\n },\n ],\n)\n\nunion(tables: [static, dynamic])","selected":["standalone"],"type":"query"}}]