diff --git a/salt/telegraf/config.sls b/salt/telegraf/config.sls index a35be55f5..dea76cd82 100644 --- a/salt/telegraf/config.sls +++ b/salt/telegraf/config.sls @@ -45,6 +45,24 @@ tgraf_sync_script_{{script}}: GLOBALS: {{ GLOBALS }} {% endfor %} +{% if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managersearch', 'so-heavynode', 'so-eval', 'so-import'] %} +tgraf_sync_script_esindexsize.sh: + file.managed: + - name: /opt/so/conf/telegraf/scripts/esindexsize.sh + - user: root + - group: 939 + - mode: 770 + - source: salt://telegraf/scripts/esindexsize.sh +{# Copy conf/elasticsearch/curl.config for telegraf to use with esindexsize.sh #} +tgraf_sync_escurl_conf: + file.managed: + - name: /opt/so/conf/telegraf/etc/escurl.config + - user: 939 + - group: 939 + - mode: 400 + - source: /opt/so/conf/elasticsearch/curl.config +{% endif %} + telegraf_sbin: file.recurse: - name: /usr/sbin diff --git a/salt/telegraf/enabled.sls b/salt/telegraf/enabled.sls index 8c71ecac3..ffb45ceb9 100644 --- a/salt/telegraf/enabled.sls +++ b/salt/telegraf/enabled.sls @@ -56,6 +56,9 @@ so-telegraf: - /opt/so/log/sostatus:/var/log/sostatus:ro - /opt/so/log/salt:/var/log/salt:ro - /opt/so/log/agents:/var/log/agents:ro + {% if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managersearch', 'so-heavynode', 'so-eval', 'so-import'] %} + - /opt/so/conf/telegraf/etc/escurl.config:/etc/telegraf/elasticsearch.config:ro + {% endif %} {% if DOCKER.containers['so-telegraf'].custom_bind_mounts %} {% for BIND in DOCKER.containers['so-telegraf'].custom_bind_mounts %} - {{ BIND }} diff --git a/salt/telegraf/etc/telegraf.conf b/salt/telegraf/etc/telegraf.conf index 4c2318c02..a4173a014 100644 --- a/salt/telegraf/etc/telegraf.conf +++ b/salt/telegraf/etc/telegraf.conf @@ -199,6 +199,20 @@ username = "{{ ES_USER }}" password = "{{ ES_PASS }}" insecure_skip_verify = true + # Every hour collect current size of all indices +[[ inputs.elasticsearch ]] + servers = ["https://{{ NODEIP }}:9200"] + username = "{{ ES_USER }}" + password = "{{ ES_PASS }}" + insecure_skip_verify = true + + indices_level = "indices" + indices_include = ["_all"] + # Drop everything except specific field + fieldinclude = ["store_size_in_bytes"] + + interval = "1m" + {%- elif grains['role'] in ['so-searchnode'] %} [[inputs.elasticsearch]] servers = ["https://{{ NODEIP }}:9200"] @@ -323,3 +337,13 @@ # # Read metrics about network interface usage [[inputs.net]] + +# Scripts run every 30s||TELEGRAFMERGED.config.interval - ES index script doesn't need to run as frequently +{%- if grains.role in ['so-standalone', 'so-manager', 'so-managersearch', 'so-heavynode', 'so-eval', 'so-import'] %} +[[ inputs.exec ]] + commands = [ + "/scripts/esindexsize.sh" + ] + data_format = "influx" + interval = "1h" +{%- endif %} diff --git a/salt/telegraf/scripts/esindexsize.sh b/salt/telegraf/scripts/esindexsize.sh new file mode 100644 index 000000000..2ba46149e --- /dev/null +++ b/salt/telegraf/scripts/esindexsize.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +DATASTREAM_INFO=$(curl -K /etc/telegraf/elasticsearch.config -s -k -L "https://localhost:9200/_data_stream?format=json") +INDICES=$(curl -K /etc/telegraf/elasticsearch.config -s -k -L "https://localhost:9200/_cat/indices?h=index,store.size&bytes=b&s=index:asc&format=json") +INDICES_WITH_SIZE=() + +while IFS= read -r DS; do + datastream_indices=() + datastream=$(echo "$DS" | jq -r '.name') + # influx doesn't like key starting with '.' + if [[ $datastream != .* ]]; then + while IFS= read -r DS_IDX; do + datastream_indices+=("$DS_IDX") + done < <(echo "$DS" | jq -r '.indices[].index_name') + datastream_size=0 + + for idx in ${datastream_indices[@]}; do + current_index=$(echo "$INDICES" | jq -r --arg index "$idx" '.[] | select(.index == $index)["store.size"]') + datastream_size=$(($datastream_size + $current_index)) + done + INDICES_WITH_SIZE+=("${datastream}=${datastream_size}i") + # echo "$datastream size is $(echo "$datastream_size" | numfmt --to iec)" + fi +done < <(echo "$DATASTREAM_INFO" | jq -c '.data_streams[]') + +measurement="elasticsearch_index_size " +total=${#INDICES_WITH_SIZE[@]} +for idxws in "${!INDICES_WITH_SIZE[@]}"; do + if [[ $idxws -lt $(($total - 1)) ]]; then + measurement+="${INDICES_WITH_SIZE[$idxws]}," + else + measurement+="${INDICES_WITH_SIZE[$idxws]}" + fi +done + +echo "$measurement" \ No newline at end of file