influx upgrade

salt/common/tools/sbin/so-influxdb-manage

@@ -0,0 +1,238 @@
+#!/bin/bash
+
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+. /usr/sbin/so-common
+
+usage() {
+    echo "Usage: $0 <operation> [args]"
+    echo ""
+    echo "Supported Operations:"
+    echo "  setup          Loads all templates and creates all required buckets"
+    echo "  templateapply  Applies a single template file, requires: <template-path>"
+    echo "  userlist       Lists users"
+    echo "  useradd        Adds a new user, requires: <email>"
+    echo "  userdel        Removes an existing user, requires: <email>"
+    echo "  userenable     Enables a user, requires: <email>"
+    echo "  userdisable    Disables a user, requires: <email>"
+    echo "  userpass       Updates a user's password, requires: <email>"
+    echo ""
+    echo "If required, the password will be read from STDIN."
+    exit 1
+}
+
+if [ $# -lt 1 ]; then
+  usage
+fi
+
+OP=$1
+shift
+
+set -eo pipefail
+
+log() {
+  echo -e "$(date) | InfluxDB | $@" >&2
+}
+
+check_response() {
+  response=$1
+  if [[ "$response" =~ "\"code\":" ]]; then
+    log "Failed. Check the response for more details.\n$response"
+    exit 1
+  fi
+}
+
+lookup_user_id() {
+  token=$1
+  email=$2
+  response=$(curl -sk https://localhost:8086/api/v2/users?limit=100 -H "Authorization: Token $token")
+  check_response "$response"
+  uid=$(echo "$response" | jq -r ".users[] | select(.name == \"$email\").id")
+  if [[ -z "$uid" ]]; then
+    log "User not found"
+    exit 1
+  fi
+  echo "$uid"
+}
+
+lookup_org_id() {
+  token=$1
+  response=$(curl -sk https://localhost:8086/api/v2/orgs?limit=100 -H "Authorization: Token $token")
+  check_response "$response"
+  oid=$(echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id")
+  if [[ -z "$oid" ]]; then
+    log "Organization not found"
+    exit 1
+  fi
+  echo "$oid"
+}
+
+lookup_stack_id() {
+  token=$1
+  oid=$2
+  response=$(curl -sk "https://localhost:8086/api/v2/stacks?orgID=$oid&name=Security+Onion" -H "Authorization: Token $token")
+  check_response "$response"
+  stackid=$(echo "$response" | jq -r ".stacks[0].id")
+  if [[ -z "$stackid" || "$stackid" == null ]]; then
+    response=$(curl -sk https://localhost:8086/api/v2/stacks -X POST -d "{\"name\":\"Security Onion\",\"orgID\":\"$oid\"}" -H "Authorization: Token $token")
+    check_response "$response"
+    stackid=$(echo "$response" | jq -r .id)
+  fi
+  echo "$stackid"
+}
+
+add_user_to_org() {
+  token=$1
+  uid=$2
+  oid=$3
+  log "Adding new user to organization"
+  response=$(curl -sk https://localhost:8086/api/v2/orgs/$oid/members -X POST -d "{\"id\":\"$uid\"}" -H "Authorization: Token $token")
+  check_response "$response"
+}
+
+change_password() {
+  token=$1
+  uid=$2
+  set +e
+  test -t 0
+  if [[ $? == 0 ]]; then
+    echo "Enter new password:"
+  fi
+  set -e
+  read -rs pass
+  check_password_and_exit "$pass"
+  response=$(curl -sk https://localhost:8086/api/v2/users/$uid/password -X POST -d "{\"password\":\"$pass\"}" -H "Authorization: Token $token")
+  check_response "$response"
+}
+
+apply_template() {
+  token=$1
+  oid=$2
+  stackid=$3
+  file=$4
+  content=$(cat $file)
+  body="{\"orgID\":\"$oid\",\"stackID\":\"$stackid\",\"template\":{\"contents\":$content}}"
+  response=$(curl -sk https://localhost:8086/api/v2/templates/apply -X POST -d "$body" -H "Authorization: Token $token")
+  check_response "$response"
+}
+
+create_bucket() {
+  token=$1
+  oid=$2
+  name=$3
+  age=$4
+  shardduration=$5
+
+  response=$(curl -sk "https://localhost:8086/api/v2/buckets?orgID=$oid&name=$name" -H "Authorization: Token $token")
+  check_response "$response"
+  bucketid=$(echo "$response" | jq -r ".buckets[0].id")
+  if [[ -z "$stackid" || "$stackid" == null ]]; then
+    response=$(curl -sk https://localhost:8086/api/v2/buckets -X POST -d "{\"name\":\"$name\",\"orgID\":\"oid\"}" -H "Authorization: Token $token")
+    check_response "$response"
+    bucketid=$(echo "$response" | jq -r .id)
+  fi
+  response=$(curl -sk "https://localhost:8086/api/v2/buckets/$bucketid" -d "{\"name\":\"$name\",\"retentionRules\":[{\"everySeconds\":$age,\"shardGroupDurationSeconds\":$shardduration,\"type\":\"expire\"}]}" -H "Authorization: Token $token")
+  check_response "$response"
+}
+
+case "$OP" in
+
+  templateload)
+    [ $# -ne 1 ] && usage
+    file=$1
+    log "Applying template file; file=$file"
+    token=$(lookup_pillar_secret influx_token)
+    oid=$(lookup_org_id "$token")
+    stackid=$(lookup_stack_id "$token" "$oid")
+    apply_template "$token" "$oid" "$stackid" "$file"
+    ;;
+
+  setup)
+    log "Ensuring organization is setup correctly"
+    token=$(lookup_pillar_secret influx_token)
+    oid=$(lookup_org_id "$token")
+
+    # Load templates
+    stackid=$(lookup_stack_id "$token" "$oid")
+    for file in /opt/so/conf/influxdb/templates/*; do
+      log "Ensuring template is loaded; template=$file"
+      apply_template "$token" "$oid" "$stackid" "$file"
+    done
+
+    # Setup buckets and retention periods
+    for rp in so_short_term so_long_term; do
+      bucket=telegraf/$rp
+      log "Ensuring bucket is created and configured; bucket=$bucket"
+      age=$(cat /opt/so/conf/influxdb/buckets.json | jq -r .$rp.duration)
+      shard_duration=$(cat /opt/so/conf/influxdb/buckets.json | jq -r .$rp.shard_duration)
+      create_bucket "$token" "$oid" "$bucket" "$age" "$shard_duration"
+    done
+    ;;    
+
+  userlist)
+    log "Listing existing users"
+    token=$(lookup_pillar_secret influx_token)
+    response=$(curl -sk https://localhost:8086/api/v2/users -H "Authorization: Token $token")
+    check_response "$response"
+    echo "$response" | jq -r '.users[] | "\(.id): \(.name) (\(.status))"'
+    ;;
+
+  useradd)
+    [ $# -ne 1 ] && usage
+    email=$1
+    log "Adding new user; email=$email"
+    token=$(lookup_pillar_secret influx_token)
+    oid=$(lookup_org_id "$token")
+    response=$(curl -sk https://localhost:8086/api/v2/users -X POST -d "{\"name\":\"$email\"}" -H "Authorization: Token $token")
+    check_response "$response"
+    uid=$(echo "$response" | jq -r .id)
+    add_user_to_org "$token" "$uid" "$oid"
+    change_password "$token" "$uid"
+    ;;
+
+  userpass)
+    [ $# -ne 1 ] && usage
+    email=$1
+    log "Updating user password; email=$email"
+    token=$(lookup_pillar_secret influx_token)
+    uid=$(lookup_user_id "$token" "$email")
+    change_password "$token" "$uid"
+    ;;
+  
+  userdel)
+    [ $# -ne 1 ] && usage
+    email=$1
+    log "Deleting user; email=$email"
+    token=$(lookup_pillar_secret influx_token)
+    uid=$(lookup_user_id "$token" "$email")
+    response=$(curl -sk https://localhost:8086/api/v2/users/$uid -X DELETE -H "Authorization: Token $token")
+    check_response "$response"
+    ;;
+
+  userenable)
+    [ $# -ne 1 ] && usage
+    email=$1
+    log "Enabling user; email=$email"
+    token=$(lookup_pillar_secret influx_token)
+    uid=$(lookup_user_id "$token" "$email")
+    response=$(curl -sk https://localhost:8086/api/v2/users/$uid -X PATCH -d "{\"name\":\"$email\",\"status\":\"active\"}" -H "Authorization: Token $token")
+    check_response "$response"
+    ;;
+
+  userdisable)
+    [ $# -ne 1 ] && usage
+    email=$1
+    log "Disabling user; email=$email"
+    token=$(lookup_pillar_secret influx_token)
+    uid=$(lookup_user_id "$token" "$email")
+    response=$(curl -sk https://localhost:8086/api/v2/users/$uid -X PATCH -d "{\"name\":\"$email\",\"status\":\"inactive\"}" -H "Authorization: Token $token")
+    check_response "$response"
+    ;;
+
+  *)
+    usage
+    ;;
+esac

salt/common/tools/sbin/so-influxdb-user

@@ -1,114 +0,0 @@
-#!/bin/bash
-
-# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
-# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at 
-# https://securityonion.net/license; you may not use this file except in compliance with the
-# Elastic License 2.0.
-
-. /usr/sbin/so-common
-
-usage() {
-    echo "Usage: $0 <add|delete|enable|disable|password> <new-user-email>"
-    echo ""
-    echo "Supported Operations:"
-    echo "  add        Adds a new user"
-    echo "  delete     Removes an existing user"
-    echo "  enable     Enables a user"
-    echo "  disable    Disables a user"
-    echo "  password   Updates a user's password"
-    echo ""
-    echo "If required, the password will be read from STDIN."
-    exit 1
-}
-
-if [ $# -ne 2 ]; then
-  usage
-fi
-
-KIND=InfluxDB
-OP=$1
-USER_EMAIL=$2
-TOKEN=$(lookup_pillar_secret influx_token)
-
-log() {
-  echo -e "$@"
-}
-
-read_password() {
-  # Read password for new user from stdin
-  set +e
-  test -t 0
-  if [[ $? == 0 ]]; then
-    echo "Enter new password:"
-  fi
-  set -e
-  read -rs USER_PASS
-
-  check_password_and_exit "$USER_PASS"
-}
-
-check_response() {
-  response=$1
-  if [[ "$response" =~ "\"code\":" ]]; then
-    log "Failed. Check the response for more details.\n$response"
-    exit 1
-  fi
-}
-
-set -eo pipefail
-
-if [[ "$OP" == "add" ]]; then
-  log "Creating new $KIND user"
-  response=$(curl -sk https://localhost:8086/api/v2/users -X POST -d "{\"name\":\"$USER_EMAIL\"}" -H "Authorization: Token $TOKEN")
-  check_response "$response"
-fi
-
-log "Looking up user ID"
-response=$(curl -sk https://localhost:8086/api/v2/users?limit=100 -H "Authorization: Token $TOKEN")
-check_response "$response"
-USER_ID=$(echo "$response" | jq -r ".users[] | select(.name == \"$USER_EMAIL\").id")
-if [[ -z "$USER_ID" ]]; then
-  log "$KIND user not found"
-  exit 1
-fi
-
-log "Looking up organization ID"
-response=$(curl -sk https://localhost:8086/api/v2/orgs?limit=100 -H "Authorization: Token $TOKEN")
-check_response "$response"
-ORG_ID=$(echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id")
-if [[ -z "$ORG_ID" ]]; then
-  log "$KIND organization not found"
-  exit 1
-fi
-
-if [[ "$OP" == "add" ]]; then
-  log "Adding new $KIND user to organization"
-  response=$(curl -sk https://localhost:8086/api/v2/orgs/$ORG_ID/members -X POST -d "{\"id\":\"$USER_ID\"}" -H "Authorization: Token $TOKEN")
-  check_response "$response"
-  OP=password
-fi
-
-if [[ "$OP" == "password" ]]; then
-  read_password
-  log "Updating $KIND user password"
-  response=$(curl -sk https://localhost:8086/api/v2/users/$USER_ID/password -X POST -d "{\"password\":\"$USER_PASS\"}" -H "Authorization: Token $TOKEN")
-  check_response "$response"
-fi
-
-if [[ "$OP" == "delete" ]]; then
-  log "Deleting $KIND user"
-  response=$(curl -sk https://localhost:8086/api/v2/users/$USER_ID -X DELETE -H "Authorization: Token $TOKEN")
-  check_response "$response"
-fi
-
-if [[ "$OP" == "enable" ]]; then
-  log "Enabling $KIND user"
-  response=$(curl -sk https://localhost:8086/api/v2/users/$USER_ID -X PATCH -d "{\"name\":\"$USER_EMAIL\",\"status\":\"active\"}" -H "Authorization: Token $TOKEN")
-  check_response "$response"
-fi
-
-if [[ "$OP" == "disable" ]]; then
-  log "Disabling $KIND user"
-  response=$(curl -sk https://localhost:8086/api/v2/users/$USER_ID -X PATCH -d "{\"name\":\"$USER_EMAIL\",\"status\":\"inactive\"}" -H "Authorization: Token $TOKEN")
-  check_response "$response"
-fi

salt/common/tools/sbin/so-user

@@ -587,8 +587,7 @@ case "${operation}" in
     createUser "$email" "${role:-$DEFAULT_ROLE}" "${firstName}" "${lastName}" "${note}"
     syncAll
     echo "Successfully added new user to SOC"
-    check_container fleet && echo "$password" | so-fleet-user-add "$email"
+    echo "$password" | so-influxdb-manage useradd "$email"
-    echo "$password" | so-influxdb-user add "$email"
     ;;
 
   "list")
@@ -629,7 +628,7 @@ case "${operation}" in
     updateUserPassword "$email"
     syncAll
     echo "Successfully updated user password"
-    echo "$password" | so-influxdb-user password "$email"
+    echo "$password" | so-influxdb-manage userpass "$email"
     ;;
 
   "profile")
@@ -649,8 +648,7 @@ case "${operation}" in
     updateStatus "$email" 'active'
     syncAll
     echo "Successfully enabled user"
-    echo "Fleet user will need to be recreated manually with so-fleet-user-add"
+    so-influxdb-manage userenable "$email"
-    so-influxdb-user enable "$email"
     ;;
 
   "disable")
@@ -661,8 +659,7 @@ case "${operation}" in
     updateStatus "$email" 'locked'
     syncAll
     echo "Successfully disabled user"
-    check_container fleet && so-fleet-user-delete "$email"   
+    so-influxdb-manage userdisable "$email"
-    so-influxdb-user disable "$email"
     ;;    
 
   "delete")
@@ -673,8 +670,7 @@ case "${operation}" in
     deleteUser "$email"
     syncAll
     echo "Successfully deleted user"
-    check_container fleet && so-fleet-user-delete "$email"
+    so-influxdb-manage userdel "$email"
-    so-influxdb-user delete "$email"
     ;;
 
   "sync")

salt/influxdb/buckets.json.jinja

@@ -0,0 +1,3 @@
+{%- from 'influxdb/map.jinja' import INFLUXMERGED %}
+
+{{ INFLUXMERGED.buckets | json }}

salt/influxdb/config.yaml.jinja

@@ -1,4 +1,3 @@
-{%- import_yaml 'influxdb/defaults.yaml' as INFLUXDEFAULTS %}
+{%- from 'influxdb/map.jinja' import INFLUXMERGED %}
-{%- set INFLUXMERGED = salt['pillar.get']('influxdb:config', default=INFLUXDEFAULTS.influxdb.config, merge=true) %}
 
-{{ INFLUXMERGED | yaml(false) }}
+{{ INFLUXMERGED.config | yaml(false) }}

salt/influxdb/defaults.yaml

@@ -65,15 +65,13 @@ influxdb:
     vault-skip-verify: false
     vault-tls-server-name: ""
     vault-token: ""
-  retention_policies:
+  buckets:
     so_short_term:
-      default: True
+      duration: 2592000
-      duration: 30d
+      shard_duration: 86400
-      shard_duration: 1d
     so_long_term:
-      default: False
+      duration: 0
-      duration: 0d
+      shard_duration: 604800
-      shard_duration: 7d
   downsample:
     so_long_term:
       resolution: 5m

salt/influxdb/init.sls

@@ -33,10 +33,31 @@ influxdbdir:
 influxdbconf:
   file.managed:
     - name: /opt/so/conf/influxdb/config.yaml
+    - source: salt://influxdb/config.yaml.jinja
     - user: 939
     - group: 939
     - template: jinja
-    - source: salt://influxdb/config.yaml.jinja
+
+influxdbbucketsconf:
+  file.managed:
+    - name: /opt/so/conf/influxdb/buckets.json
+    - source: salt://influxdb/buckets.json.jinja
+    - user: 939
+    - group: 939
+    - template: jinja
+
+influxdb-templates:
+  file.recurse:
+    - name: /opt/so/conf/influxdb/templates
+    - source: salt://influxdb/templates
+    - user: 939
+    - group: 939
+    - template: jinja
+    - clean: True
+
+influxdb-setup:
+  cmd.run:
+    - name: /usr/sbin/so-influxdb-setup
 
 so-influxdb:
   docker_container.running:
@@ -53,7 +74,6 @@ so-influxdb:
       - DOCKER_INFLUXDB_INIT_PASSWORD={{ PASSWORD }}
       - DOCKER_INFLUXDB_INIT_ORG=Security Onion
       - DOCKER_INFLUXDB_INIT_BUCKET=telegraf/so_short_term
-      - DOCKER_INFLUXDB_INIT_RETENTION=30d
       - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN={{ TOKEN }}
     - binds:
       - /opt/so/log/influxdb/:/log:rw

salt/influxdb/map.jinja

@@ -1,9 +1,2 @@
-{% import_yaml 'influxdb/defaults.yaml' as INFLUXDB %}
+{%- import_yaml 'influxdb/defaults.yaml' as INFLUXDEFAULTS %}
-{% set measurements = salt['cmd.shell']('docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -database telegraf -execute "show measurements" 2> /root/measurement_query.log | jq -r .results[0].series[0].values[]?[0] 2>> /root/measurement_query.log', shell='/bin/bash') %}
+{%- set INFLUXMERGED = salt['pillar.get']('influxdb', default=INFLUXDEFAULTS.influxdb, merge=true) %}
-
-{% if measurements|length > 0 %}
-{%   do INFLUXDB.influxdb.downsample.so_long_term.update('measurements': [])%}
-{%   for measurement in measurements.splitlines() %}
-{%     do INFLUXDB.influxdb.downsample.so_long_term.measurements.append(measurement)%}
-{%   endfor %}
-{% endif %}

salt/influxdb/soc_influxdb.yaml

@@ -328,23 +328,23 @@ influxdb:
       global: True
       advanced: True
       helpLink: influxdb.html
-  retention_policies:
+  buckets:
     so_short_term:
       duration:
-        description: Amount of time to keep short term data.
+        description: Amount of time (in seconds) to keep short term data.
         global: True
         helpLink: influxdb.html
       shard_duration:
-        description: Time range 
+        description: Amount of the time (in seconds) range covered by the shard group.
         global: True
         helpLink: influxdb.html
     so_long_term:
       duration:
-        description: Amount of time to keep long term downsampled data.
+        description: Amount of time (in seconds) to keep long term downsampled data.
         global: True
         helpLink: influxdb.html
       shard_duration:
-        description: Amount of the time range covered by the shard group.
+        description: Amount of the time (in seconds) range covered by the shard group.
         global: True
         helpLink: influxdb.html
   downsample:

salt/soc/defaults.map.jinja

@@ -1,6 +1,7 @@
 {% import_yaml 'soc/defaults.yaml' as SOCDEFAULTS %}
 {% from 'vars/globals.map.jinja' import GLOBALS %}
 {% from 'docker/docker.map.jinja' import DOCKER -%}
+{%- set INFLUXDB_TOKEN = salt['pillar.get']('secrets:influx_token') %}
 
 {% for module, application_url in GLOBALS.application_urls.items() %}
 {%   do SOCDEFAULTS.soc.server.modules[module].update({'hostUrl': application_url}) %}
@@ -15,9 +16,8 @@
 
 {% do SOCDEFAULTS.soc.server.modules.elastic.update({'username': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'password': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}) %}
 
-{% if GLOBALS.role != 'so-import' %}
 {% do SOCDEFAULTS.soc.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %}
-{% endif %}
+{% do SOCDEFAULTS.soc.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %}
 
 {% do SOCDEFAULTS.soc.server.modules.statickeyauth.update({'anonymousCidr': DOCKER.sorange, 'apiKey': pillar.sensoroni.sensoronikey}) %}
 

salt/soc/defaults.yaml

@@ -1031,9 +1031,9 @@ soc:
         asyncThreshold: 10
       influxdb:
         hostUrl:
-        token: ''
+        token: 
-        org: ''
+        org: Security Onion
-        bucket: telegraf
+        bucket: telegraf/so_short_term
         verifyCert: false
       salt:
         saltPipe: /opt/sensoroni/salt/pipe