diff --git a/salt/common/tools/sbin/so-influxdb-manage b/salt/common/tools/sbin/so-influxdb-manage new file mode 100644 index 000000000..e5f480369 --- /dev/null +++ b/salt/common/tools/sbin/so-influxdb-manage @@ -0,0 +1,238 @@ +#!/bin/bash + +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +. /usr/sbin/so-common + +usage() { + echo "Usage: $0 [args]" + echo "" + echo "Supported Operations:" + echo " setup Loads all templates and creates all required buckets" + echo " templateapply Applies a single template file, requires: " + echo " userlist Lists users" + echo " useradd Adds a new user, requires: " + echo " userdel Removes an existing user, requires: " + echo " userenable Enables a user, requires: " + echo " userdisable Disables a user, requires: " + echo " userpass Updates a user's password, requires: " + echo "" + echo "If required, the password will be read from STDIN." + exit 1 +} + +if [ $# -lt 1 ]; then + usage +fi + +OP=$1 +shift + +set -eo pipefail + +log() { + echo -e "$(date) | InfluxDB | $@" >&2 +} + +check_response() { + response=$1 + if [[ "$response" =~ "\"code\":" ]]; then + log "Failed. Check the response for more details.\n$response" + exit 1 + fi +} + +lookup_user_id() { + token=$1 + email=$2 + response=$(curl -sk https://localhost:8086/api/v2/users?limit=100 -H "Authorization: Token $token") + check_response "$response" + uid=$(echo "$response" | jq -r ".users[] | select(.name == \"$email\").id") + if [[ -z "$uid" ]]; then + log "User not found" + exit 1 + fi + echo "$uid" +} + +lookup_org_id() { + token=$1 + response=$(curl -sk https://localhost:8086/api/v2/orgs?limit=100 -H "Authorization: Token $token") + check_response "$response" + oid=$(echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id") + if [[ -z "$oid" ]]; then + log "Organization not found" + exit 1 + fi + echo "$oid" +} + +lookup_stack_id() { + token=$1 + oid=$2 + response=$(curl -sk "https://localhost:8086/api/v2/stacks?orgID=$oid&name=Security+Onion" -H "Authorization: Token $token") + check_response "$response" + stackid=$(echo "$response" | jq -r ".stacks[0].id") + if [[ -z "$stackid" || "$stackid" == null ]]; then + response=$(curl -sk https://localhost:8086/api/v2/stacks -X POST -d "{\"name\":\"Security Onion\",\"orgID\":\"$oid\"}" -H "Authorization: Token $token") + check_response "$response" + stackid=$(echo "$response" | jq -r .id) + fi + echo "$stackid" +} + +add_user_to_org() { + token=$1 + uid=$2 + oid=$3 + log "Adding new user to organization" + response=$(curl -sk https://localhost:8086/api/v2/orgs/$oid/members -X POST -d "{\"id\":\"$uid\"}" -H "Authorization: Token $token") + check_response "$response" +} + +change_password() { + token=$1 + uid=$2 + set +e + test -t 0 + if [[ $? == 0 ]]; then + echo "Enter new password:" + fi + set -e + read -rs pass + check_password_and_exit "$pass" + response=$(curl -sk https://localhost:8086/api/v2/users/$uid/password -X POST -d "{\"password\":\"$pass\"}" -H "Authorization: Token $token") + check_response "$response" +} + +apply_template() { + token=$1 + oid=$2 + stackid=$3 + file=$4 + content=$(cat $file) + body="{\"orgID\":\"$oid\",\"stackID\":\"$stackid\",\"template\":{\"contents\":$content}}" + response=$(curl -sk https://localhost:8086/api/v2/templates/apply -X POST -d "$body" -H "Authorization: Token $token") + check_response "$response" +} + +create_bucket() { + token=$1 + oid=$2 + name=$3 + age=$4 + shardduration=$5 + + response=$(curl -sk "https://localhost:8086/api/v2/buckets?orgID=$oid&name=$name" -H "Authorization: Token $token") + check_response "$response" + bucketid=$(echo "$response" | jq -r ".buckets[0].id") + if [[ -z "$stackid" || "$stackid" == null ]]; then + response=$(curl -sk https://localhost:8086/api/v2/buckets -X POST -d "{\"name\":\"$name\",\"orgID\":\"oid\"}" -H "Authorization: Token $token") + check_response "$response" + bucketid=$(echo "$response" | jq -r .id) + fi + response=$(curl -sk "https://localhost:8086/api/v2/buckets/$bucketid" -d "{\"name\":\"$name\",\"retentionRules\":[{\"everySeconds\":$age,\"shardGroupDurationSeconds\":$shardduration,\"type\":\"expire\"}]}" -H "Authorization: Token $token") + check_response "$response" +} + +case "$OP" in + + templateload) + [ $# -ne 1 ] && usage + file=$1 + log "Applying template file; file=$file" + token=$(lookup_pillar_secret influx_token) + oid=$(lookup_org_id "$token") + stackid=$(lookup_stack_id "$token" "$oid") + apply_template "$token" "$oid" "$stackid" "$file" + ;; + + setup) + log "Ensuring organization is setup correctly" + token=$(lookup_pillar_secret influx_token) + oid=$(lookup_org_id "$token") + + # Load templates + stackid=$(lookup_stack_id "$token" "$oid") + for file in /opt/so/conf/influxdb/templates/*; do + log "Ensuring template is loaded; template=$file" + apply_template "$token" "$oid" "$stackid" "$file" + done + + # Setup buckets and retention periods + for rp in so_short_term so_long_term; do + bucket=telegraf/$rp + log "Ensuring bucket is created and configured; bucket=$bucket" + age=$(cat /opt/so/conf/influxdb/buckets.json | jq -r .$rp.duration) + shard_duration=$(cat /opt/so/conf/influxdb/buckets.json | jq -r .$rp.shard_duration) + create_bucket "$token" "$oid" "$bucket" "$age" "$shard_duration" + done + ;; + + userlist) + log "Listing existing users" + token=$(lookup_pillar_secret influx_token) + response=$(curl -sk https://localhost:8086/api/v2/users -H "Authorization: Token $token") + check_response "$response" + echo "$response" | jq -r '.users[] | "\(.id): \(.name) (\(.status))"' + ;; + + useradd) + [ $# -ne 1 ] && usage + email=$1 + log "Adding new user; email=$email" + token=$(lookup_pillar_secret influx_token) + oid=$(lookup_org_id "$token") + response=$(curl -sk https://localhost:8086/api/v2/users -X POST -d "{\"name\":\"$email\"}" -H "Authorization: Token $token") + check_response "$response" + uid=$(echo "$response" | jq -r .id) + add_user_to_org "$token" "$uid" "$oid" + change_password "$token" "$uid" + ;; + + userpass) + [ $# -ne 1 ] && usage + email=$1 + log "Updating user password; email=$email" + token=$(lookup_pillar_secret influx_token) + uid=$(lookup_user_id "$token" "$email") + change_password "$token" "$uid" + ;; + + userdel) + [ $# -ne 1 ] && usage + email=$1 + log "Deleting user; email=$email" + token=$(lookup_pillar_secret influx_token) + uid=$(lookup_user_id "$token" "$email") + response=$(curl -sk https://localhost:8086/api/v2/users/$uid -X DELETE -H "Authorization: Token $token") + check_response "$response" + ;; + + userenable) + [ $# -ne 1 ] && usage + email=$1 + log "Enabling user; email=$email" + token=$(lookup_pillar_secret influx_token) + uid=$(lookup_user_id "$token" "$email") + response=$(curl -sk https://localhost:8086/api/v2/users/$uid -X PATCH -d "{\"name\":\"$email\",\"status\":\"active\"}" -H "Authorization: Token $token") + check_response "$response" + ;; + + userdisable) + [ $# -ne 1 ] && usage + email=$1 + log "Disabling user; email=$email" + token=$(lookup_pillar_secret influx_token) + uid=$(lookup_user_id "$token" "$email") + response=$(curl -sk https://localhost:8086/api/v2/users/$uid -X PATCH -d "{\"name\":\"$email\",\"status\":\"inactive\"}" -H "Authorization: Token $token") + check_response "$response" + ;; + + *) + usage + ;; +esac diff --git a/salt/common/tools/sbin/so-influxdb-user b/salt/common/tools/sbin/so-influxdb-user deleted file mode 100644 index 90cf84e25..000000000 --- a/salt/common/tools/sbin/so-influxdb-user +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash - -# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at -# https://securityonion.net/license; you may not use this file except in compliance with the -# Elastic License 2.0. - -. /usr/sbin/so-common - -usage() { - echo "Usage: $0 " - echo "" - echo "Supported Operations:" - echo " add Adds a new user" - echo " delete Removes an existing user" - echo " enable Enables a user" - echo " disable Disables a user" - echo " password Updates a user's password" - echo "" - echo "If required, the password will be read from STDIN." - exit 1 -} - -if [ $# -ne 2 ]; then - usage -fi - -KIND=InfluxDB -OP=$1 -USER_EMAIL=$2 -TOKEN=$(lookup_pillar_secret influx_token) - -log() { - echo -e "$@" -} - -read_password() { - # Read password for new user from stdin - set +e - test -t 0 - if [[ $? == 0 ]]; then - echo "Enter new password:" - fi - set -e - read -rs USER_PASS - - check_password_and_exit "$USER_PASS" -} - -check_response() { - response=$1 - if [[ "$response" =~ "\"code\":" ]]; then - log "Failed. Check the response for more details.\n$response" - exit 1 - fi -} - -set -eo pipefail - -if [[ "$OP" == "add" ]]; then - log "Creating new $KIND user" - response=$(curl -sk https://localhost:8086/api/v2/users -X POST -d "{\"name\":\"$USER_EMAIL\"}" -H "Authorization: Token $TOKEN") - check_response "$response" -fi - -log "Looking up user ID" -response=$(curl -sk https://localhost:8086/api/v2/users?limit=100 -H "Authorization: Token $TOKEN") -check_response "$response" -USER_ID=$(echo "$response" | jq -r ".users[] | select(.name == \"$USER_EMAIL\").id") -if [[ -z "$USER_ID" ]]; then - log "$KIND user not found" - exit 1 -fi - -log "Looking up organization ID" -response=$(curl -sk https://localhost:8086/api/v2/orgs?limit=100 -H "Authorization: Token $TOKEN") -check_response "$response" -ORG_ID=$(echo "$response" | jq -r ".orgs[] | select(.name == \"Security Onion\").id") -if [[ -z "$ORG_ID" ]]; then - log "$KIND organization not found" - exit 1 -fi - -if [[ "$OP" == "add" ]]; then - log "Adding new $KIND user to organization" - response=$(curl -sk https://localhost:8086/api/v2/orgs/$ORG_ID/members -X POST -d "{\"id\":\"$USER_ID\"}" -H "Authorization: Token $TOKEN") - check_response "$response" - OP=password -fi - -if [[ "$OP" == "password" ]]; then - read_password - log "Updating $KIND user password" - response=$(curl -sk https://localhost:8086/api/v2/users/$USER_ID/password -X POST -d "{\"password\":\"$USER_PASS\"}" -H "Authorization: Token $TOKEN") - check_response "$response" -fi - -if [[ "$OP" == "delete" ]]; then - log "Deleting $KIND user" - response=$(curl -sk https://localhost:8086/api/v2/users/$USER_ID -X DELETE -H "Authorization: Token $TOKEN") - check_response "$response" -fi - -if [[ "$OP" == "enable" ]]; then - log "Enabling $KIND user" - response=$(curl -sk https://localhost:8086/api/v2/users/$USER_ID -X PATCH -d "{\"name\":\"$USER_EMAIL\",\"status\":\"active\"}" -H "Authorization: Token $TOKEN") - check_response "$response" -fi - -if [[ "$OP" == "disable" ]]; then - log "Disabling $KIND user" - response=$(curl -sk https://localhost:8086/api/v2/users/$USER_ID -X PATCH -d "{\"name\":\"$USER_EMAIL\",\"status\":\"inactive\"}" -H "Authorization: Token $TOKEN") - check_response "$response" -fi diff --git a/salt/common/tools/sbin/so-user b/salt/common/tools/sbin/so-user index 46feef640..da6f3a4fb 100755 --- a/salt/common/tools/sbin/so-user +++ b/salt/common/tools/sbin/so-user @@ -587,8 +587,7 @@ case "${operation}" in createUser "$email" "${role:-$DEFAULT_ROLE}" "${firstName}" "${lastName}" "${note}" syncAll echo "Successfully added new user to SOC" - check_container fleet && echo "$password" | so-fleet-user-add "$email" - echo "$password" | so-influxdb-user add "$email" + echo "$password" | so-influxdb-manage useradd "$email" ;; "list") @@ -629,7 +628,7 @@ case "${operation}" in updateUserPassword "$email" syncAll echo "Successfully updated user password" - echo "$password" | so-influxdb-user password "$email" + echo "$password" | so-influxdb-manage userpass "$email" ;; "profile") @@ -649,8 +648,7 @@ case "${operation}" in updateStatus "$email" 'active' syncAll echo "Successfully enabled user" - echo "Fleet user will need to be recreated manually with so-fleet-user-add" - so-influxdb-user enable "$email" + so-influxdb-manage userenable "$email" ;; "disable") @@ -661,8 +659,7 @@ case "${operation}" in updateStatus "$email" 'locked' syncAll echo "Successfully disabled user" - check_container fleet && so-fleet-user-delete "$email" - so-influxdb-user disable "$email" + so-influxdb-manage userdisable "$email" ;; "delete") @@ -673,8 +670,7 @@ case "${operation}" in deleteUser "$email" syncAll echo "Successfully deleted user" - check_container fleet && so-fleet-user-delete "$email" - so-influxdb-user delete "$email" + so-influxdb-manage userdel "$email" ;; "sync") diff --git a/salt/influxdb/buckets.json.jinja b/salt/influxdb/buckets.json.jinja new file mode 100644 index 000000000..1e4655d31 --- /dev/null +++ b/salt/influxdb/buckets.json.jinja @@ -0,0 +1,3 @@ +{%- from 'influxdb/map.jinja' import INFLUXMERGED %} + +{{ INFLUXMERGED.buckets | json }} \ No newline at end of file diff --git a/salt/influxdb/config.yaml.jinja b/salt/influxdb/config.yaml.jinja index 720c4967c..50b8ff940 100644 --- a/salt/influxdb/config.yaml.jinja +++ b/salt/influxdb/config.yaml.jinja @@ -1,4 +1,3 @@ -{%- import_yaml 'influxdb/defaults.yaml' as INFLUXDEFAULTS %} -{%- set INFLUXMERGED = salt['pillar.get']('influxdb:config', default=INFLUXDEFAULTS.influxdb.config, merge=true) %} +{%- from 'influxdb/map.jinja' import INFLUXMERGED %} -{{ INFLUXMERGED | yaml(false) }} \ No newline at end of file +{{ INFLUXMERGED.config | yaml(false) }} \ No newline at end of file diff --git a/salt/influxdb/defaults.yaml b/salt/influxdb/defaults.yaml index 82e59db90..983844faf 100644 --- a/salt/influxdb/defaults.yaml +++ b/salt/influxdb/defaults.yaml @@ -65,15 +65,13 @@ influxdb: vault-skip-verify: false vault-tls-server-name: "" vault-token: "" - retention_policies: + buckets: so_short_term: - default: True - duration: 30d - shard_duration: 1d + duration: 2592000 + shard_duration: 86400 so_long_term: - default: False - duration: 0d - shard_duration: 7d + duration: 0 + shard_duration: 604800 downsample: so_long_term: resolution: 5m \ No newline at end of file diff --git a/salt/influxdb/init.sls b/salt/influxdb/init.sls index a3a59db70..96b1e7b76 100644 --- a/salt/influxdb/init.sls +++ b/salt/influxdb/init.sls @@ -33,10 +33,31 @@ influxdbdir: influxdbconf: file.managed: - name: /opt/so/conf/influxdb/config.yaml + - source: salt://influxdb/config.yaml.jinja - user: 939 - group: 939 - template: jinja - - source: salt://influxdb/config.yaml.jinja + +influxdbbucketsconf: + file.managed: + - name: /opt/so/conf/influxdb/buckets.json + - source: salt://influxdb/buckets.json.jinja + - user: 939 + - group: 939 + - template: jinja + +influxdb-templates: + file.recurse: + - name: /opt/so/conf/influxdb/templates + - source: salt://influxdb/templates + - user: 939 + - group: 939 + - template: jinja + - clean: True + +influxdb-setup: + cmd.run: + - name: /usr/sbin/so-influxdb-setup so-influxdb: docker_container.running: @@ -53,7 +74,6 @@ so-influxdb: - DOCKER_INFLUXDB_INIT_PASSWORD={{ PASSWORD }} - DOCKER_INFLUXDB_INIT_ORG=Security Onion - DOCKER_INFLUXDB_INIT_BUCKET=telegraf/so_short_term - - DOCKER_INFLUXDB_INIT_RETENTION=30d - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN={{ TOKEN }} - binds: - /opt/so/log/influxdb/:/log:rw diff --git a/salt/influxdb/map.jinja b/salt/influxdb/map.jinja index a13d4c257..ecbdd1306 100644 --- a/salt/influxdb/map.jinja +++ b/salt/influxdb/map.jinja @@ -1,9 +1,2 @@ -{% import_yaml 'influxdb/defaults.yaml' as INFLUXDB %} -{% set measurements = salt['cmd.shell']('docker exec -t so-influxdb influx -format json -ssl -unsafeSsl -database telegraf -execute "show measurements" 2> /root/measurement_query.log | jq -r .results[0].series[0].values[]?[0] 2>> /root/measurement_query.log', shell='/bin/bash') %} - -{% if measurements|length > 0 %} -{% do INFLUXDB.influxdb.downsample.so_long_term.update('measurements': [])%} -{% for measurement in measurements.splitlines() %} -{% do INFLUXDB.influxdb.downsample.so_long_term.measurements.append(measurement)%} -{% endfor %} -{% endif %} +{%- import_yaml 'influxdb/defaults.yaml' as INFLUXDEFAULTS %} +{%- set INFLUXMERGED = salt['pillar.get']('influxdb', default=INFLUXDEFAULTS.influxdb, merge=true) %} diff --git a/salt/influxdb/soc_influxdb.yaml b/salt/influxdb/soc_influxdb.yaml index cbef9db85..7f6ceb316 100644 --- a/salt/influxdb/soc_influxdb.yaml +++ b/salt/influxdb/soc_influxdb.yaml @@ -328,23 +328,23 @@ influxdb: global: True advanced: True helpLink: influxdb.html - retention_policies: + buckets: so_short_term: duration: - description: Amount of time to keep short term data. + description: Amount of time (in seconds) to keep short term data. global: True helpLink: influxdb.html shard_duration: - description: Time range + description: Amount of the time (in seconds) range covered by the shard group. global: True helpLink: influxdb.html so_long_term: duration: - description: Amount of time to keep long term downsampled data. + description: Amount of time (in seconds) to keep long term downsampled data. global: True helpLink: influxdb.html shard_duration: - description: Amount of the time range covered by the shard group. + description: Amount of the time (in seconds) range covered by the shard group. global: True helpLink: influxdb.html downsample: diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index e26a8050f..912b16f46 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -1,6 +1,7 @@ {% import_yaml 'soc/defaults.yaml' as SOCDEFAULTS %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER -%} +{%- set INFLUXDB_TOKEN = salt['pillar.get']('secrets:influx_token') %} {% for module, application_url in GLOBALS.application_urls.items() %} {% do SOCDEFAULTS.soc.server.modules[module].update({'hostUrl': application_url}) %} @@ -15,9 +16,8 @@ {% do SOCDEFAULTS.soc.server.modules.elastic.update({'username': GLOBALS.elasticsearch.auth.users.so_elastic_user.user, 'password': GLOBALS.elasticsearch.auth.users.so_elastic_user.pass}) %} -{% if GLOBALS.role != 'so-import' %} -{% do SOCDEFAULTS.soc.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %} -{% endif %} +{% do SOCDEFAULTS.soc.server.modules.influxdb.update({'hostUrl': 'https://' ~ GLOBALS.influxdb_host ~ ':8086'}) %} +{% do SOCDEFAULTS.soc.server.modules.influxdb.update({'token': INFLUXDB_TOKEN}) %} {% do SOCDEFAULTS.soc.server.modules.statickeyauth.update({'anonymousCidr': DOCKER.sorange, 'apiKey': pillar.sensoroni.sensoronikey}) %} diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index b83b94493..0723f2da9 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1031,9 +1031,9 @@ soc: asyncThreshold: 10 influxdb: hostUrl: - token: '' - org: '' - bucket: telegraf + token: + org: Security Onion + bucket: telegraf/so_short_term verifyCert: false salt: saltPipe: /opt/sensoroni/salt/pipe