add metrics for es index sizes

salt/telegraf/config.sls

@@ -45,6 +45,24 @@ tgraf_sync_script_{{script}}:
         GLOBALS: {{ GLOBALS }}
 {% endfor %}
 
+{% if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managersearch', 'so-heavynode', 'so-eval', 'so-import'] %}
+tgraf_sync_script_esindexsize.sh:
+  file.managed:
+    - name: /opt/so/conf/telegraf/scripts/esindexsize.sh
+    - user: root
+    - group: 939
+    - mode: 770
+    - source: salt://telegraf/scripts/esindexsize.sh
+{# Copy conf/elasticsearch/curl.config for telegraf to use with esindexsize.sh #}
+tgraf_sync_escurl_conf:
+  file.managed:
+    - name: /opt/so/conf/telegraf/etc/escurl.config
+    - user: 939
+    - group: 939
+    - mode: 400
+    - source: /opt/so/conf/elasticsearch/curl.config
+{% endif %}
+
 telegraf_sbin:
   file.recurse:
     - name: /usr/sbin

salt/telegraf/enabled.sls

@@ -56,6 +56,9 @@ so-telegraf:
       - /opt/so/log/sostatus:/var/log/sostatus:ro
       - /opt/so/log/salt:/var/log/salt:ro
       - /opt/so/log/agents:/var/log/agents:ro
+      {% if GLOBALS.role in ['so-standalone', 'so-manager', 'so-managersearch', 'so-heavynode', 'so-eval', 'so-import'] %}
+      - /opt/so/conf/telegraf/etc/escurl.config:/etc/telegraf/elasticsearch.config:ro
+      {% endif %}
       {% if DOCKER.containers['so-telegraf'].custom_bind_mounts %}
         {% for BIND in DOCKER.containers['so-telegraf'].custom_bind_mounts %}
       - {{ BIND }}

salt/telegraf/etc/telegraf.conf

@@ -199,6 +199,20 @@
   username = "{{ ES_USER }}"
   password = "{{ ES_PASS }}"
   insecure_skip_verify = true
+  # Every hour collect current size of all indices
+[[ inputs.elasticsearch ]]
+  servers = ["https://{{ NODEIP }}:9200"]
+  username = "{{ ES_USER }}"
+  password = "{{ ES_PASS }}"
+  insecure_skip_verify = true
+
+  indices_level = "indices"
+  indices_include = ["_all"]
+  # Drop everything except specific field
+  fieldinclude = ["store_size_in_bytes"]
+
+  interval = "1m"
+
 {%- elif grains['role'] in ['so-searchnode']  %}
 [[inputs.elasticsearch]]
   servers = ["https://{{ NODEIP }}:9200"]
@@ -323,3 +337,13 @@
 
 # # Read metrics about network interface usage
  [[inputs.net]]
+
+# Scripts run every 30s||TELEGRAFMERGED.config.interval - ES index script doesn't need to run as frequently
+{%- if grains.role in ['so-standalone', 'so-manager', 'so-managersearch', 'so-heavynode', 'so-eval', 'so-import'] %}
+[[ inputs.exec ]]
+  commands = [
+    "/scripts/esindexsize.sh"
+  ]
+  data_format = "influx"
+  interval = "1h"
+{%- endif %}

salt/telegraf/scripts/esindexsize.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one
+# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at
+# https://securityonion.net/license; you may not use this file except in compliance with the
+# Elastic License 2.0.
+
+DATASTREAM_INFO=$(curl -K /etc/telegraf/elasticsearch.config -s -k -L "https://localhost:9200/_data_stream?format=json")
+INDICES=$(curl -K /etc/telegraf/elasticsearch.config -s -k -L "https://localhost:9200/_cat/indices?h=index,store.size&bytes=b&s=index:asc&format=json")
+INDICES_WITH_SIZE=()
+
+while IFS= read -r DS; do
+    datastream_indices=()
+    datastream=$(echo "$DS" | jq -r '.name')
+    # influx doesn't like key starting with '.'
+    if [[ $datastream != .* ]]; then
+        while IFS= read -r DS_IDX; do
+            datastream_indices+=("$DS_IDX")
+        done < <(echo "$DS" | jq -r '.indices[].index_name')
+        datastream_size=0
+
+        for idx in ${datastream_indices[@]}; do
+            current_index=$(echo "$INDICES" | jq -r --arg index "$idx" '.[] | select(.index == $index)["store.size"]')
+            datastream_size=$(($datastream_size + $current_index))
+        done
+        INDICES_WITH_SIZE+=("${datastream}=${datastream_size}i")
+        # echo "$datastream size is $(echo "$datastream_size" | numfmt --to iec)"
+    fi
+done < <(echo "$DATASTREAM_INFO" | jq -c '.data_streams[]')
+
+measurement="elasticsearch_index_size "
+total=${#INDICES_WITH_SIZE[@]}
+for idxws in "${!INDICES_WITH_SIZE[@]}"; do
+    if [[ $idxws -lt $(($total - 1)) ]]; then
+        measurement+="${INDICES_WITH_SIZE[$idxws]},"
+    else
+        measurement+="${INDICES_WITH_SIZE[$idxws]}"
+    fi
+done
+
+echo "$measurement"