Merge pull request #6301 from Security-Onion-Solutions/dev

2.3.90

CONTRIBUTING.md

@@ -15,7 +15,7 @@
 
 ### Contributing code
 
-* **All commits must be signed** with a valid key that has been added to your GitHub account. The commits should have all the "**Verified**" tag when viewed on GitHub as shown below:
+* **All commits must be signed** with a valid key that has been added to your GitHub account. Each commit should have the "**Verified**" tag when viewed on GitHub as shown below:
   
   <img src="./assets/images/verified-commit-1.png" width="450">
 

README.md

@@ -1,6 +1,6 @@
-## Security Onion 2.3.80
+## Security Onion 2.3.90
 
-Security Onion 2.3.80 is here!
+Security Onion 2.3.90 is here!
 
 ## Screenshots
 

VERIFY_ISO.md

@@ -1,18 +1,18 @@
-### 2.3.80 ISO image built on 2021/09/27
+### 2.3.90 ISO image built on 2021/11/19
 
 
 
 ### Download and Verify
 
-2.3.80 ISO image:  
+2.3.90 ISO image:  
-https://download.securityonion.net/file/securityonion/securityonion-2.3.80.iso
+https://download.securityonion.net/file/securityonion/securityonion-2.3.90.iso
 
-MD5: 24F38563860416F4A8ABE18746913E14  
+MD5: F214ECE9F32A6F881D9A735DEAF90E46  
-SHA1: F923C005F54EA2A17AB225ADA0DA46042707AAD9  
+SHA1: 0B04FAA0FEC704CF6AD2030AA7A4AE80D9379AFA  
-SHA256: 8E95D10AF664D9A406C168EC421D943CB23F0D0C1813C6C2DBA9B4E131984018 
+SHA256: BE0E1516D83D7782AEAE9D52449FED45A45D72981515672C761C2A17B7AA613C 
 
 Signature for ISO image:  
-https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.80.iso.sig
+https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.90.iso.sig
 
 Signing key:  
 https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/master/KEYS  
@@ -26,22 +26,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/ma
 
 Download the signature file for the ISO:  
 ```
-wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.80.iso.sig
+wget https://github.com/Security-Onion-Solutions/securityonion/raw/master/sigs/securityonion-2.3.90.iso.sig
 ```
 
 Download the ISO image:  
 ```
-wget https://download.securityonion.net/file/securityonion/securityonion-2.3.80.iso
+wget https://download.securityonion.net/file/securityonion/securityonion-2.3.90.iso
 ```
 
 Verify the downloaded ISO image using the signature file:  
 ```
-gpg --verify securityonion-2.3.80.iso.sig securityonion-2.3.80.iso
+gpg --verify securityonion-2.3.90.iso.sig securityonion-2.3.90.iso
 ```
 
 The output should show "Good signature" and the Primary key fingerprint should match what's shown below:
 ```
-gpg: Signature made Mon 27 Sep 2021 08:55:01 AM EDT using RSA key ID FE507013
+gpg: Signature made Fri 19 Nov 2021 05:15:29 PM EST using RSA key ID FE507013
 gpg: Good signature from "Security Onion Solutions, LLC <info@securityonionsolutions.com>"
 gpg: WARNING: This key is not certified with a trusted signature!
 gpg:          There is no indication that the signature belongs to the owner.

VERSION

@@ -1 +1 @@
-2.3.80
+2.3.90

files/firewall/hostgroups.local.yaml

@@ -16,6 +16,10 @@ firewall:
       ips:
         delete:
         insert:
+    endgame:
+      ips:
+        delete:
+        insert:
     fleet:
       ips:
         delete:

pillar/elasticsearch/manager.sls

@@ -2,6 +2,7 @@ elasticsearch:
   templates:
     - so/so-beats-template.json.jinja
     - so/so-common-template.json.jinja
+    - so/so-endgame-template.json.jinja
     - so/so-firewall-template.json.jinja
     - so/so-flow-template.json.jinja
     - so/so-ids-template.json.jinja

pillar/elasticsearch/search.sls

@@ -2,6 +2,7 @@ elasticsearch:
   templates:
     - so/so-beats-template.json.jinja
     - so/so-common-template.json.jinja
+    - so/so-endgame-template.json.jinja
     - so/so-firewall-template.json.jinja
     - so/so-flow-template.json.jinja
     - so/so-ids-template.json.jinja

pillar/logstash/init.sls

@@ -1,6 +1,7 @@
 logstash:
   docker_options:
     port_bindings:
+      - 0.0.0.0:3765:3765
       - 0.0.0.0:5044:5044
       - 0.0.0.0:5644:5644
       - 0.0.0.0:6050:6050

pillar/logstash/manager.sls

@@ -5,5 +5,6 @@ logstash:
       config:
         - so/0009_input_beats.conf      
         - so/0010_input_hhbeats.conf
+        - so/0011_input_endgame.conf
         - so/9999_output_redis.conf.jinja
         

pillar/logstash/search.sls

@@ -14,3 +14,4 @@ logstash:
         - so/9600_output_ossec.conf.jinja
         - so/9700_output_strelka.conf.jinja
         - so/9800_output_logscan.conf.jinja
+        - so/9900_output_endgame.conf.jinja

pillar/top.sls

@@ -24,6 +24,9 @@ base:
     - data.*
 {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
+{% endif %}
+{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
+    - kibana.secrets
 {% endif %}
     - secrets
     - global
@@ -43,6 +46,9 @@ base:
     - elasticsearch.eval
 {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
+{% endif %}
+{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
+    - kibana.secrets
 {% endif %}
     - global
     - minions.{{ grains.id }}
@@ -54,6 +60,9 @@ base:
     - elasticsearch.search
 {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
+{% endif %}
+{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
+    - kibana.secrets
 {% endif %}
     - data.*
     - zeeklogs
@@ -101,6 +110,9 @@ base:
     - elasticsearch.eval
 {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %}
     - elasticsearch.auth
+{% endif %}
+{% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/kibana/secrets.sls') %}
+    - kibana.secrets
 {% endif %}
     - global
     - minions.{{ grains.id }}

salt/allowed_states.map.jinja

@@ -35,6 +35,7 @@
           'influxdb',
           'grafana',
           'soc',
+          'kratos',
           'firewall',
           'idstools',
           'suricata.manager',
@@ -100,6 +101,7 @@
           'manager',
           'nginx',
           'soc',
+          'kratos',
           'firewall',
           'idstools',
           'suricata.manager',
@@ -123,6 +125,7 @@
           'influxdb',
           'grafana',
           'soc',
+          'kratos',
           'firewall',
           'idstools',
           'suricata.manager',
@@ -142,6 +145,7 @@
           'influxdb',
           'grafana',
           'soc',
+          'kratos',
           'firewall',
           'manager',
           'idstools',
@@ -172,6 +176,7 @@
           'influxdb',
           'grafana',
           'soc',
+          'kratos',
           'firewall',
           'idstools',
           'suricata.manager',
@@ -238,8 +243,13 @@
     {% do allowed_states.append('elasticsearch') %}
   {% endif %}
 
+  {% if ELASTICSEARCH and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
+    {% do allowed_states.append('elasticsearch.auth') %}
+  {% endif %}
+
   {% if KIBANA and grains.role in ['so-eval', 'so-manager', 'so-standalone', 'so-managersearch', 'so-import'] %}
     {% do allowed_states.append('kibana') %}
+    {% do allowed_states.append('kibana.secrets') %}
   {% endif %}
 
   {% if grains.role in ['so-eval', 'so-standalone', 'so-node', 'so-managersearch', 'so-heavynode', 'so-manager'] %}

salt/ca/init.sls

@@ -24,8 +24,9 @@ pki_private_key:
       - x509: /etc/pki/ca.crt
     {%- endif %}
 
-/etc/pki/ca.crt:
+pki_public_ca_crt:
   x509.certificate_managed:
+    - name: /etc/pki/ca.crt
     - signing_private_key: /etc/pki/ca.key
     - CN: {{ manager }}
     - C: US
@@ -66,4 +67,4 @@ cakeyperms:
   test.fail_without_changes:
     - name: {{sls}}_state_not_allowed
 
-{% endif %}
+{% endif %}

salt/common/init.sls

@@ -9,6 +9,11 @@ rmvariablesfile:
   file.absent:
     - name: /tmp/variables.txt
 
+dockergroup:
+  group.present:
+    - name: docker
+    - gid: 920
+
 # Add socore Group
 socoregroup:
   group.present:
@@ -101,16 +106,24 @@ commonpkgs:
       - python3-m2crypto
       - python3-mysqldb
       - python3-packaging
+      - python3-lxml
       - git
       - vim
 
 heldpackages:
   pkg.installed:
     - pkgs:
+    {% if grains['oscodename'] == 'bionic' %}
       - containerd.io: 1.4.4-1
       - docker-ce: 5:20.10.5~3-0~ubuntu-bionic
       - docker-ce-cli: 5:20.10.5~3-0~ubuntu-bionic
       - docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-bionic
+    {% elif grains['oscodename'] == 'focal' %}
+      - containerd.io: 1.4.9-1
+      - docker-ce: 5:20.10.8~3-0~ubuntu-focal
+      - docker-ce-cli: 5:20.10.5~3-0~ubuntu-focal
+      - docker-ce-rootless-extras: 5:20.10.5~3-0~ubuntu-focal
+    {% endif %}
     - hold: True
     - update_holds: True
 
@@ -136,6 +149,7 @@ commonpkgs:
       - python36-m2crypto
       - python36-mysql
       - python36-packaging
+      - python36-lxml
       - yum-utils
       - device-mapper-persistent-data
       - lvm2

salt/common/tools/sbin/so-allow

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env python3
 
 # Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
@@ -15,152 +15,199 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-. /usr/sbin/so-common
+import ipaddress
+import textwrap
+import os
+import subprocess
+import sys
+import argparse
+import re
+from lxml import etree as ET
+from xml.dom import minidom
+from datetime import datetime as dt
+from datetime import timezone as tz
 
-local_salt_dir=/opt/so/saltstack/local
-
-SKIP=0
-
-function usage {
-
-cat << EOF
-
-Usage: $0 [-abefhoprsw] [ -i IP ]
-
-This program allows you to add a firewall rule to allow connections from a new IP address or CIDR range.
-
-If you run this program with no arguments, it will present a menu for you to choose your options.
-
-If you want to automate and skip the menu, you can pass the desired options as command line arguments.
-
-EXAMPLES
-
-To add 10.1.2.3 to the analyst role:
-so-allow -a -i 10.1.2.3
-
-To add 10.1.2.0/24 to the osquery role:
-so-allow -o -i 10.1.2.0/24
-
-EOF
 
+LOCAL_SALT_DIR='/opt/so/saltstack/local'
+WAZUH_CONF='/nsm/wazuh/etc/ossec.conf'
+VALID_ROLES = {
+  'a': { 'role': 'analyst','desc': 'Analyst - 80/tcp, 443/tcp' },
+  'b': { 'role': 'beats_endpoint', 'desc': 'Logstash Beat - 5044/tcp' },
+  'e': { 'role': 'elasticsearch_rest', 'desc': 'Elasticsearch REST API - 9200/tcp' },
+  'f': { 'role': 'strelka_frontend', 'desc': 'Strelka frontend - 57314/tcp' },
+  'o': { 'role': 'osquery_endpoint', 'desc': 'Osquery endpoint - 8090/tcp' },
+  's': { 'role': 'syslog', 'desc': 'Syslog device - 514/tcp/udp' },
+  'w': { 'role': 'wazuh_agent', 'desc': 'Wazuh agent - 1514/tcp/udp' },
+  'p': { 'role': 'wazuh_api', 'desc': 'Wazuh API - 55000/tcp' },
+  'r': { 'role': 'wazuh_authd', 'desc': 'Wazuh registration service - 1515/tcp' }
 }
 
-while getopts "ahfesprbowi:" OPTION
-do
-    case $OPTION in
-        h)
-            usage
-            exit 0
-            ;;
-        a)
-            FULLROLE="analyst"
-            SKIP=1
-            ;;
-        b)
-            FULLROLE="beats_endpoint"
-            SKIP=1
-            ;;
-	e)
-            FULLROLE="elasticsearch_rest"
-            SKIP=1
-            ;;
-        f)
-	    FULLROLE="strelka_frontend"
-            SKIP=1
-            ;;
-        i)  IP=$OPTARG
-            ;;
-        o)
-            FULLROLE="osquery_endpoint"
-            SKIP=1
-            ;;
-        w)
-            FULLROLE="wazuh_agent"
-            SKIP=1
-            ;;
-        s)
-            FULLROLE="syslog"
-            SKIP=1
-            ;;
-        p)
-            FULLROLE="wazuh_api"
-            SKIP=1
-            ;;
-        r)
-            FULLROLE="wazuh_authd"
-            SKIP=1
-            ;;
-        *)
-            usage
-            exit 0
-            ;;
-    esac
-done
 
-if [ "$SKIP" -eq 0 ]; then
+def validate_ip_cidr(ip_cidr: str) -> bool:
+  try:
+    ipaddress.ip_address(ip_cidr)
+  except ValueError:
+    try:
+      ipaddress.ip_network(ip_cidr)
+    except ValueError:
+      return False
+  return True
 
-    echo "This program allows you to add a firewall rule to allow connections from a new IP address."
-    echo ""
-    echo "Choose the role for the IP or Range you would like to add"
-    echo ""
-    echo "[a] - Analyst - ports 80/tcp and 443/tcp"
-    echo "[b] - Logstash Beat - port 5044/tcp"
-    echo "[e] - Elasticsearch REST API - port 9200/tcp"
-    echo "[f] - Strelka frontend - port 57314/tcp"
-    echo "[o] - Osquery endpoint - port 8090/tcp"
-    echo "[s] - Syslog device - 514/tcp/udp"
-    echo "[w] - Wazuh agent - port 1514/tcp/udp"
-    echo "[p] - Wazuh API - port 55000/tcp"
-    echo "[r] - Wazuh registration service - 1515/tcp"
-    echo ""
-    echo "Please enter your selection:"
-    read -r ROLE
-    echo "Enter a single ip address or range to allow (example: 10.10.10.10 or 10.10.0.0/16):"
-    read -r IP
 
-    if [ "$ROLE" == "a" ]; then
+def role_prompt() -> str:
-        FULLROLE=analyst
+  print()
-    elif [ "$ROLE" == "b" ]; then
+  print('Choose the role for the IP or Range you would like to allow')
-        FULLROLE=beats_endpoint
+  print()
-    elif [ "$ROLE" == "e" ]; then
+  for role in VALID_ROLES:
-        FULLROLE=elasticsearch_rest
+    print(f'[{role}] - {VALID_ROLES[role]["desc"]}')
-    elif [ "$ROLE" == "f" ]; then
+  print()
-        FULLROLE=strelka_frontend
+  role = input('Please enter your selection: ')
-    elif [ "$ROLE" == "o" ]; then
+  if role in VALID_ROLES.keys():
-        FULLROLE=osquery_endpoint
+    return VALID_ROLES[role]['role']
-    elif [ "$ROLE" == "w" ]; then
+  else:
-        FULLROLE=wazuh_agent
+    print(f'Invalid role \'{role}\', please try again.', file=sys.stderr)
-    elif [ "$ROLE" == "s" ]; then
+    sys.exit(1)
-        FULLROLE=syslog
+  
-    elif [ "$ROLE" == "p" ]; then
-        FULLROLE=wazuh_api
-    elif [ "$ROLE" == "r" ]; then
-        FULLROLE=wazuh_authd
-    else
-        echo "I don't recognize that role"
-        exit 1
-    fi
 
-fi
+def ip_prompt() -> str:
+  ip = input('Enter a single ip address or range to allow (ex: 10.10.10.10 or 10.10.0.0/16): ')
+  if validate_ip_cidr(ip):
+    return ip
+  else:
+    print(f'Invalid IP address or CIDR block \'{ip}\', please try again.', file=sys.stderr)
+    sys.exit(1)
 
-echo "Adding $IP to the $FULLROLE role. This can take a few seconds"
-/usr/sbin/so-firewall includehost $FULLROLE $IP
-salt-call state.apply firewall queue=True
 
-# Check if Wazuh enabled
+def wazuh_enabled() -> bool:
-if grep -q -R "wazuh: 1" $local_salt_dir/pillar/*; then
+  for file in os.listdir(f'{LOCAL_SALT_DIR}/pillar'):
-    # If analyst, add to Wazuh AR whitelist
+    with open(file, 'r') as pillar:
-    if [ "$FULLROLE" == "analyst" ]; then
+      if 'wazuh: 1' in pillar.read():
-        WAZUH_MGR_CFG="/nsm/wazuh/etc/ossec.conf"
+        return True
-        if ! grep -q "<white_list>$IP</white_list>" $WAZUH_MGR_CFG ; then
+  return False
-            DATE=$(date)
+
-            sed -i 's/<\/ossec_config>//' $WAZUH_MGR_CFG
+
-            sed -i '/^$/N;/^\n$/D' $WAZUH_MGR_CFG
+def root_to_str(root: ET.ElementTree) -> str:
-            echo -e "<!--Address $IP added by /usr/sbin/so-allow on \"$DATE\"-->\n  <global>\n    <white_list>$IP</white_list>\n  </global>\n</ossec_config>" >> $WAZUH_MGR_CFG
+    xml_str = ET.tostring(root, encoding='unicode', method='xml').replace('\n', '')
-            echo "Added whitelist entry for $IP in $WAZUH_MGR_CFG."
+    xml_str = re.sub(r'(?:(?<=>) *)', '', xml_str)
-        echo
+    xml_str = re.sub(r'       -', '', xml_str)
-            echo "Restarting OSSEC Server..."
+    xml_str = re.sub(r'      -->', ' -->', xml_str)
-            /usr/sbin/so-wazuh-restart
+    dom = minidom.parseString(xml_str)
-        fi
+    return dom.toprettyxml(indent="  ")
-    fi
+
-fi
+
+def add_wl(ip):
+    parser = ET.XMLParser(remove_blank_text=True)
+    with open(WAZUH_CONF, 'rb') as wazuh_conf:
+        tree = ET.parse(wazuh_conf, parser)
+    root = tree.getroot()
+
+    source_comment = ET.Comment(f'Address {ip} added by /usr/sbin/so-allow on {dt.utcnow().replace(tzinfo=tz.utc).strftime("%a %b %e %H:%M:%S %Z %Y")}')
+    new_global = ET.Element("global")
+    new_wl = ET.SubElement(new_global, 'white_list')
+    new_wl.text = ip
+
+    root.append(source_comment)
+    root.append(new_global)
+
+    with open(WAZUH_CONF, 'w') as add_out:
+        add_out.write(root_to_str(root))
+
+
+def apply(role: str, ip: str) -> int:
+  firewall_cmd = ['so-firewall', 'includehost', role, ip]
+  salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', 'firewall', 'queue=True']
+  restart_wazuh_cmd = ['so-wazuh-restart']
+  print(f'Adding {ip} to the {role} role. This can take a few seconds...')
+  cmd = subprocess.run(firewall_cmd)
+  if cmd.returncode == 0:
+    cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL)
+  else:
+    return cmd.returncode
+  if cmd.returncode == 0:
+    if wazuh_enabled and role=='analyst':
+      try:
+        add_wl(ip)
+        print(f'Added whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr)
+      except Exception as e:
+        print(f'Failed to add whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr)
+        print(e)
+        return 1
+      print('Restarting OSSEC Server...')
+      cmd = subprocess.run(restart_wazuh_cmd)
+    else:
+      return cmd.returncode
+  else:
+    print(f'Commmand \'{" ".join(salt_cmd)}\' failed.', file=sys.stderr)
+    return cmd.returncode
+  if cmd.returncode != 0:
+    print('Failed to restart OSSEC server.')
+  return cmd.returncode
+
+
+def main():
+  if os.geteuid() != 0:
+    print('You must run this script as root', file=sys.stderr)
+    sys.exit(1)
+
+  main_parser = argparse.ArgumentParser(
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    epilog=textwrap.dedent(f'''\
+        additional information:
+                      To use this script in interactive mode call it with no arguments
+        '''
+  ))
+
+  group = main_parser.add_argument_group(title='roles')
+  group.add_argument('-a', dest='roles', action='append_const', const=VALID_ROLES['a']['role'], help="Analyst - 80/tcp, 443/tcp")
+  group.add_argument('-b', dest='roles', action='append_const', const=VALID_ROLES['b']['role'], help="Logstash Beat - 5044/tcp")
+  group.add_argument('-e', dest='roles', action='append_const', const=VALID_ROLES['e']['role'], help="Elasticsearch REST API - 9200/tcp")
+  group.add_argument('-f', dest='roles', action='append_const', const=VALID_ROLES['f']['role'], help="Strelka frontend - 57314/tcp")
+  group.add_argument('-o', dest='roles', action='append_const', const=VALID_ROLES['o']['role'], help="Osquery endpoint - 8090/tcp")
+  group.add_argument('-s', dest='roles', action='append_const', const=VALID_ROLES['s']['role'], help="Syslog device - 514/tcp/udp")
+  group.add_argument('-w', dest='roles', action='append_const', const=VALID_ROLES['w']['role'], help="Wazuh agent - 1514/tcp/udp")
+  group.add_argument('-p', dest='roles', action='append_const', const=VALID_ROLES['p']['role'], help="Wazuh API - 55000/tcp")
+  group.add_argument('-r', dest='roles', action='append_const', const=VALID_ROLES['r']['role'], help="Wazuh registration service - 1515/tcp")
+  
+  ip_g = main_parser.add_argument_group(title='allow')
+  ip_g.add_argument('-i', help="IP or CIDR block to disallow connections from, requires at least one role argument", metavar='', dest='ip')
+
+  args = main_parser.parse_args(sys.argv[1:])
+
+  if args.roles is None:
+    role = role_prompt()
+    ip = ip_prompt()
+    try:
+      return_code = apply(role, ip)
+    except Exception as e:
+      print(f'Unexpected exception occurred: {e}', file=sys.stderr)
+      return_code = e.errno
+    sys.exit(return_code)
+  elif args.roles is not None and args.ip is None:
+    if os.environ.get('IP') is None:
+      main_parser.print_help()
+      sys.exit(1)
+    else:
+      args.ip = os.environ['IP']
+  
+  if validate_ip_cidr(args.ip):
+    try:
+      for role in args.roles:
+        return_code = apply(role, args.ip)
+        if return_code > 0:
+          break
+    except Exception as e:
+      print(f'Unexpected exception occurred: {e}', file=sys.stderr)
+      return_code = e.errno
+  else:
+    print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr)
+    return_code = 1
+    
+  sys.exit(return_code)
+
+
+if __name__ == '__main__':
+  try:
+    main()
+  except KeyboardInterrupt:
+    sys.exit(1)
+

salt/common/tools/sbin/so-common

@@ -392,15 +392,18 @@ has_uppercase() {
 valid_cidr() {
 	# Verify there is a backslash in the string
 	echo "$1" | grep -qP "^[^/]+/[^/]+$" || return 1
-	
-	local cidr
-	local ip
 
-	cidr=$(echo "$1" | sed 's/.*\///')
+	valid_ip4_cidr_mask "$1" && return 0 || return 1
-	ip=$(echo "$1" | sed 's/\/.*//' )
+	
+	local cidr="$1"
+	local ip
+	ip=$(echo "$cidr" | sed 's/\/.*//' )
 	
 	if valid_ip4 "$ip"; then
-		[[ $cidr =~ ([0-9]|[1-2][0-9]|3[0-2]) ]] && return 0 || return 1
+		local ip1 ip2 ip3 ip4 N
+		IFS="./" read -r ip1 ip2 ip3 ip4 N <<< "$cidr"
+		ip_total=$((ip1 * 256 ** 3 + ip2 * 256 ** 2 + ip3 * 256 + ip4))
+		[[ $((ip_total % 2**(32-N))) == 0 ]] && return 0 || return 1
 	else
 		return 1
 	fi
@@ -450,6 +453,23 @@ valid_ip4() {
 	echo "$ip" | grep -qP '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$' && return 0 || return 1
 }
 
+valid_ip4_cidr_mask() {
+	# Verify there is a backslash in the string
+	echo "$1" | grep -qP "^[^/]+/[^/]+$" || return 1
+
+	local cidr
+	local ip
+
+	cidr=$(echo "$1" | sed 's/.*\///')
+	ip=$(echo "$1" | sed 's/\/.*//' )
+
+	if valid_ip4 "$ip"; then
+		[[ $cidr =~ ^([0-9]|[1-2][0-9]|3[0-2])$ ]] && return 0 || return 1
+	else
+		return 1
+	fi
+}
+
 valid_int() {
 	local num=$1
 	local min=${2:-1}

salt/common/tools/sbin/so-deny

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import ipaddress
+import textwrap
+import os
+import subprocess
+import sys
+import argparse
+import re
+from lxml import etree as ET
+from xml.dom import minidom
+
+
+LOCAL_SALT_DIR='/opt/so/saltstack/local'
+WAZUH_CONF='/nsm/wazuh/etc/ossec.conf'
+VALID_ROLES = {
+  'a': { 'role': 'analyst','desc': 'Analyst - 80/tcp, 443/tcp' },
+  'b': { 'role': 'beats_endpoint', 'desc': 'Logstash Beat - 5044/tcp' },
+  'e': { 'role': 'elasticsearch_rest', 'desc': 'Elasticsearch REST API - 9200/tcp' },
+  'f': { 'role': 'strelka_frontend', 'desc': 'Strelka frontend - 57314/tcp' },
+  'o': { 'role': 'osquery_endpoint', 'desc': 'Osquery endpoint - 8090/tcp' },
+  's': { 'role': 'syslog', 'desc': 'Syslog device - 514/tcp/udp' },
+  'w': { 'role': 'wazuh_agent', 'desc': 'Wazuh agent - 1514/tcp/udp' },
+  'p': { 'role': 'wazuh_api', 'desc': 'Wazuh API - 55000/tcp' },
+  'r': { 'role': 'wazuh_authd', 'desc': 'Wazuh registration service - 1515/tcp' }
+}
+
+
+def validate_ip_cidr(ip_cidr: str) -> bool:
+  try:
+    ipaddress.ip_address(ip_cidr)
+  except ValueError:
+    try:
+      ipaddress.ip_network(ip_cidr)
+    except ValueError:
+      return False
+  return True
+
+
+def role_prompt() -> str:
+  print()
+  print('Choose the role for the IP or Range you would like to deny')
+  print()
+  for role in VALID_ROLES:
+    print(f'[{role}] - {VALID_ROLES[role]["desc"]}')
+  print()
+  role = input('Please enter your selection: ')
+  if role in VALID_ROLES.keys():
+    return VALID_ROLES[role]['role']
+  else:
+    print(f'Invalid role \'{role}\', please try again.', file=sys.stderr)
+    sys.exit(1)
+  
+
+def ip_prompt() -> str:
+  ip = input('Enter a single ip address or range to deny (ex: 10.10.10.10 or 10.10.0.0/16): ')
+  if validate_ip_cidr(ip):
+    return ip
+  else:
+    print(f'Invalid IP address or CIDR block \'{ip}\', please try again.', file=sys.stderr)
+    sys.exit(1)
+
+
+def wazuh_enabled() -> bool:
+  for file in os.listdir(f'{LOCAL_SALT_DIR}/pillar'):
+    with open(file, 'r') as pillar:
+      if 'wazuh: 1' in pillar.read():
+        return True
+  return False
+
+
+def root_to_str(root: ET.ElementTree) -> str:
+    xml_str = ET.tostring(root, encoding='unicode', method='xml').replace('\n', '')
+    xml_str = re.sub(r'(?:(?<=>) *)', '', xml_str)
+
+    # Remove specific substrings to better format comments on intial parse/write
+    xml_str = re.sub(r'       -', '', xml_str)
+    xml_str = re.sub(r'      -->', ' -->', xml_str)
+    
+    dom = minidom.parseString(xml_str)
+    return dom.toprettyxml(indent="  ")
+
+
+def rem_wl(ip):
+    parser = ET.XMLParser(remove_blank_text=True)
+    with open(WAZUH_CONF, 'rb') as wazuh_conf:
+        tree = ET.parse(wazuh_conf, parser)
+    root = tree.getroot()
+
+    global_elems = root.findall(f"global/white_list[. = '{ip}']/..")
+    if len(global_elems) > 0:
+      for g_elem in global_elems:
+          ge_index = list(root).index(g_elem)
+          if ge_index > 0 and root[list(root).index(g_elem) - 1].tag == ET.Comment:
+              root.remove(root[ge_index - 1])
+          root.remove(g_elem)
+
+      with open(WAZUH_CONF, 'w') as out:
+          out.write(root_to_str(root))
+
+
+def apply(role: str, ip: str) -> int:
+  firewall_cmd = ['so-firewall', 'excludehost', role, ip]
+  salt_cmd = ['salt-call', 'state.apply', '-l', 'quiet', 'firewall', 'queue=True']
+  restart_wazuh_cmd = ['so-wazuh-restart']
+  print(f'Removing {ip} from the {role} role. This can take a few seconds...')
+  cmd = subprocess.run(firewall_cmd)
+  if cmd.returncode == 0:
+    cmd = subprocess.run(salt_cmd, stdout=subprocess.DEVNULL)
+  else:
+    return cmd.returncode
+  if cmd.returncode == 0:
+    if wazuh_enabled and role=='analyst':
+      try:
+        rem_wl(ip)
+        print(f'Removed whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr)
+      except Exception as e:
+        print(f'Failed to remove whitelist entry for {ip} from {WAZUH_CONF}', file=sys.stderr)
+        print(e)
+        return 1
+      print('Restarting OSSEC Server...')
+      cmd = subprocess.run(restart_wazuh_cmd)
+    else:
+      return cmd.returncode
+  else:
+    print(f'Commmand \'{" ".join(salt_cmd)}\' failed.', file=sys.stderr)
+    return cmd.returncode
+  if cmd.returncode != 0:
+    print('Failed to restart OSSEC server.')
+  return cmd.returncode
+
+
+def main():
+  if os.geteuid() != 0:
+    print('You must run this script as root', file=sys.stderr)
+    sys.exit(1)
+
+  main_parser = argparse.ArgumentParser(
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    epilog=textwrap.dedent(f'''\
+        additional information:
+                      To use this script in interactive mode call it with no arguments
+        '''
+  ))
+
+  group = main_parser.add_argument_group(title='roles')
+  group.add_argument('-a', dest='roles', action='append_const', const=VALID_ROLES['a']['role'], help="Analyst - 80/tcp, 443/tcp")
+  group.add_argument('-b', dest='roles', action='append_const', const=VALID_ROLES['b']['role'], help="Logstash Beat - 5044/tcp")
+  group.add_argument('-e', dest='roles', action='append_const', const=VALID_ROLES['e']['role'], help="Elasticsearch REST API - 9200/tcp")
+  group.add_argument('-f', dest='roles', action='append_const', const=VALID_ROLES['f']['role'], help="Strelka frontend - 57314/tcp")
+  group.add_argument('-o', dest='roles', action='append_const', const=VALID_ROLES['o']['role'], help="Osquery endpoint - 8090/tcp")
+  group.add_argument('-s', dest='roles', action='append_const', const=VALID_ROLES['s']['role'], help="Syslog device - 514/tcp/udp")
+  group.add_argument('-w', dest='roles', action='append_const', const=VALID_ROLES['w']['role'], help="Wazuh agent - 1514/tcp/udp")
+  group.add_argument('-p', dest='roles', action='append_const', const=VALID_ROLES['p']['role'], help="Wazuh API - 55000/tcp")
+  group.add_argument('-r', dest='roles', action='append_const', const=VALID_ROLES['r']['role'], help="Wazuh registration service - 1515/tcp")
+  
+  ip_g = main_parser.add_argument_group(title='allow')
+  ip_g.add_argument('-i', help="IP or CIDR block to disallow connections from, requires at least one role argument", metavar='', dest='ip')
+
+  args = main_parser.parse_args(sys.argv[1:])
+
+  if args.roles is None:
+    role = role_prompt()
+    ip = ip_prompt()
+    try:
+      return_code = apply(role, ip)
+    except Exception as e:
+      print(f'Unexpected exception occurred: {e}', file=sys.stderr)
+      return_code = e.errno
+    sys.exit(return_code)
+  elif args.roles is not None and args.ip is None:
+    if os.environ.get('IP') is None:
+      main_parser.print_help()
+      sys.exit(1)
+    else:
+      args.ip = os.environ['IP']
+  
+  if validate_ip_cidr(args.ip):
+    try:
+      for role in args.roles:
+        return_code = apply(role, args.ip)
+        if return_code > 0:
+          break
+    except Exception as e:
+      print(f'Unexpected exception occurred: {e}', file=sys.stderr)
+      return_code = e.errno
+  else:
+    print(f'Invalid IP address or CIDR block \'{args.ip}\', please try again.', file=sys.stderr)
+    return_code = 1
+    
+  sys.exit(return_code)
+
+
+if __name__ == '__main__':
+  try:
+    main()
+  except KeyboardInterrupt:
+    sys.exit(1)

salt/common/tools/sbin/so-elastalert-test

@@ -70,7 +70,7 @@ do
 done
 
 docker_exec(){
-	CMD="docker exec -it so-elastalert elastalert-test-rule /opt/elastalert/rules/$RULE_NAME --config /opt/config/elastalert_config.yaml $OPTIONS"
+	CMD="docker exec -it so-elastalert elastalert-test-rule /opt/elastalert/rules/$RULE_NAME --config /opt/elastalert/config.yaml $OPTIONS"
 	if [ "${RESULTS_TO_LOG,,}" = "y" ] ; then
 		$CMD > "$FILE_SAVE_LOCATION"
 	else

salt/common/tools/sbin/so-elastic-auth-password-reset

@@ -0,0 +1,155 @@
+#!/bin/bash
+
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+source $(dirname $0)/so-common
+require_manager
+
+user=$1
+elasticUsersFile=${ELASTIC_USERS_FILE:-/opt/so/saltstack/local/salt/elasticsearch/files/users}
+elasticAuthPillarFile=${ELASTIC_AUTH_PILLAR_FILE:-/opt/so/saltstack/local/pillar/elasticsearch/auth.sls}
+
+if [[ $# -ne 1 ]]; then
+  echo "Usage: $0 <user>"
+  echo ""
+  echo " where <user> is one of the following:"
+  echo ""
+  echo "         all: Reset the password for the so_elastic, so_kibana, so_logstash, so_beats, and so_monitor users"
+  echo "  so_elastic: Reset the password for the so_elastic user"
+  echo "   so_kibana: Reset the password for the so_kibana user"
+  echo " so_logstash: Reset the password for the so_logstash user"
+  echo "    so_beats: Reset the password for the so_beats user"
+  echo "  so_monitor: Reset the password for the so_monitor user"
+  echo ""
+  exit 1
+fi
+
+# function to create a lock so that the so-user sync cronjob can't run while this is running
+function lock() {
+  # Obtain file descriptor lock
+  exec 99>/var/tmp/so-user.lock || fail "Unable to create lock descriptor; if the system was not shutdown gracefully you may need to remove /var/tmp/so-user.lock manually."
+  flock -w 10 99 || fail "Another process is using so-user; if the system was not shutdown gracefully you may need to remove /var/tmp/so-user.lock manually."
+  trap 'rm -f /var/tmp/so-user.lock' EXIT
+}
+
+function unlock() {
+  rm -f /var/tmp/so-user.lock
+}
+
+function fail() {
+  msg=$1
+  echo "$1"
+  exit 1
+}
+
+function removeSingleUserPass() {
+  local user=$1
+  sed -i '/user: '"${user}"'/{N;/pass: /d}' "${elasticAuthPillarFile}"
+}
+
+function removeAllUserPass() {
+  local userList=("so_elastic" "so_kibana" "so_logstash" "so_beats" "so_monitor")
+
+  for u in ${userList[@]}; do
+    removeSingleUserPass "$u"
+  done
+}
+
+function removeElasticUsersFile() {
+  rm -f "$elasticUsersFile"
+}
+
+function createElasticAuthPillar() {
+  salt-call state.apply elasticsearch.auth queue=True
+}
+
+# this will disable highstate to prevent a highstate from starting while the script is running
+# will also disable salt.minion-state-apply-test allow so-salt-minion-check cronjob to restart salt-minion service incase
+function disableSaltStates() {
+  printf "\nDisabling salt.minion-state-apply-test and highstate from running.\n\n"
+  salt-call state.disable salt.minion-state-apply-test
+  salt-call state.disable highstate
+}
+
+function enableSaltStates() {
+  printf "\nEnabling salt.minion-state-apply-test and highstate.\n\n"
+  salt-call state.enable salt.minion-state-apply-test
+  salt-call state.enable highstate
+}
+
+function killAllSaltJobs() {
+  printf "\nKilling all running salt jobs.\n\n"
+  salt-call saltutil.kill_all_jobs
+}
+
+function soUserSync() {
+  # apply this state to update /opt/so/saltstack/local/salt/elasticsearch/curl.config on the manager
+  salt-call state.sls_id elastic_curl_config_distributed manager queue=True
+  salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' saltutil.kill_all_jobs
+  # apply this state to get the curl.config
+  salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.sls_id elastic_curl_config common queue=True
+  $(dirname $0)/so-user sync
+  printf "\nApplying logstash state to the appropriate nodes.\n\n"
+  salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.apply logstash queue=True
+  printf "\nApplying filebeat state to the appropriate nodes.\n\n"
+  salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode or G@role:so-sensor or G@role:so-fleet' state.apply filebeat queue=True
+  printf "\nApplying kibana state to the appropriate nodes.\n\n"
+  salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch' state.apply kibana queue=True
+  printf "\nApplying curator state to the appropriate nodes.\n\n"
+  salt -C 'G@role:so-standalone or G@role:so-eval or G@role:so-import or G@role:so-manager or G@role:so-managersearch or G@role:so-node or G@role:so-heavynode' state.apply curator queue=True
+}
+
+function highstateManager() {
+  killAllSaltJobs
+  printf "\nRunning highstate on the manager to finalize password reset.\n\n"
+  salt-call state.highstate -linfo queue=True
+}
+
+case "${user}" in
+
+  so_elastic | so_kibana | so_logstash | so_beats | so_monitor)
+    lock
+    killAllSaltJobs
+    disableSaltStates
+    removeSingleUserPass "$user"
+    createElasticAuthPillar
+    removeElasticUsersFile
+    unlock
+    soUserSync
+    enableSaltStates
+    highstateManager
+    ;;
+
+  all)
+    lock
+    killAllSaltJobs
+    disableSaltStates
+    removeAllUserPass
+    createElasticAuthPillar
+    removeElasticUsersFile
+    unlock
+    soUserSync
+    enableSaltStates
+    highstateManager
+    ;;
+
+  *)
+    fail "Unsupported user: $user"
+    ;;
+
+esac
+
+exit 0

salt/common/tools/sbin/so-filebeat-module-setup

@@ -54,7 +54,7 @@ PIPELINES=$({{ ELASTICCURL }} -sk https://"$ELASTICSEARCH_HOST":"$ELASTICSEARCH_
 if [[ "$PIPELINES" -lt 5 ]]; then
   echo "Setting up ingest pipeline(s)"
 
-  for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft misp mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system tomcat traefik zeek zscaler
+  for MODULE in activemq apache auditd aws azure barracuda bluecoat cef checkpoint cisco coredns crowdstrike cyberark cylance elasticsearch envoyproxy f5 fortinet gcp google_workspace googlecloud gsuite haproxy ibmmq icinga iis imperva infoblox iptables juniper kafka kibana logstash microsoft mongodb mssql mysql nats netscout nginx o365 okta osquery panw postgresql rabbitmq radware redis santa snort snyk sonicwall sophos squid suricata system threatintel tomcat traefik zeek zscaler
   do
     echo "Loading $MODULE"
     docker exec -i so-filebeat filebeat setup modules -pipelines -modules $MODULE -c $FB_MODULE_YML

salt/common/tools/sbin/so-firewall

@@ -71,7 +71,7 @@ def checkApplyOption(options):
 
 def loadYaml(filename):
   file = open(filename, "r")
-  return yaml.load(file.read())
+  return yaml.safe_load(file.read())
 
 def writeYaml(filename, content):
   file = open(filename, "w")

salt/common/tools/sbin/so-fleet-setup

@@ -2,11 +2,16 @@
 
 #so-fleet-setup $FleetEmail $FleetPassword 
 
+. /usr/sbin/so-common
+
 if [[ $# -ne 2 ]] ; then
       echo "Username or Password was not set - exiting now."
       exit 1
 fi
 
+USER_EMAIL=$1
+USER_PW=$2
+
 # Checking to see if required containers are started...
 if [ ! "$(docker ps -q -f name=so-fleet)" ]; then
         echo "Starting Docker Containers..."
@@ -17,8 +22,16 @@ fi
 
 docker exec so-fleet fleetctl config set --address https://127.0.0.1:8080 --tls-skip-verify --url-prefix /fleet
 docker exec so-fleet bash -c 'while [[ "$(curl -s -o /dev/null --insecure -w ''%{http_code}'' https://127.0.0.1:8080/fleet)" != "301" ]]; do sleep 5; done'
-docker exec so-fleet fleetctl setup --email $1 --password $2
 
+# Create Security Onion Fleet Service Account + Setup Fleet
+FLEET_SA_EMAIL=$(lookup_pillar_secret fleet_sa_email)
+FLEET_SA_PW=$(lookup_pillar_secret fleet_sa_password)
+docker exec so-fleet fleetctl setup --email $FLEET_SA_EMAIL --password $FLEET_SA_PW --name SO_ServiceAccount --org-name SO
+
+# Create User Account
+echo "$USER_PW" | so-fleet-user-add "$USER_EMAIL"
+
+# Import Packs & Configs
 docker exec so-fleet fleetctl apply -f /packs/palantir/Fleet/Endpoints/MacOS/osquery.yaml
 docker exec so-fleet fleetctl apply -f /packs/palantir/Fleet/Endpoints/Windows/osquery.yaml
 docker exec so-fleet fleetctl apply -f /packs/so/so-default.yml

salt/common/tools/sbin/so-fleet-user-add

@@ -18,7 +18,7 @@
 . /usr/sbin/so-common
 
 usage() {
-    echo "Usage: $0 <new-user-name>"
+    echo "Usage: $0 <new-user-email>"
     echo ""
     echo "Adds a new user to Fleet. The new password will be read from STDIN."
     exit 1
@@ -28,34 +28,42 @@ if [ $# -ne 1 ]; then
   usage
 fi
 
-USER=$1
 
-MYSQL_PASS=$(lookup_pillar_secret mysql)
+USER_EMAIL=$1
-FLEET_IP=$(lookup_pillar fleet_ip)
+FLEET_SA_EMAIL=$(lookup_pillar_secret fleet_sa_email)
-FLEET_USER=$USER
+FLEET_SA_PW=$(lookup_pillar_secret fleet_sa_password)
+MYSQL_PW=$(lookup_pillar_secret mysql)
 
 # Read password for new user from stdin
 test -t 0
 if [[ $? == 0 ]]; then
   echo "Enter new password:"
 fi
-read -rs FLEET_PASS
+read -rs USER_PASS
 
-check_password_and_exit "$FLEET_PASS"
+check_password_and_exit "$USER_PASS"
+
+# Config fleetctl & login with the SO Service Account
+CONFIG_OUTPUT=$(docker exec so-fleet fleetctl config set --address https://127.0.0.1:8080 --tls-skip-verify --url-prefix /fleet  2>&1 )
+SALOGIN_OUTPUT=$(docker exec so-fleet fleetctl login --email $FLEET_SA_EMAIL --password $FLEET_SA_PW 2>&1)
 
-FLEET_HASH=$(docker exec so-soctopus python -c "import bcrypt; print(bcrypt.hashpw('$FLEET_PASS'.encode('utf-8'), bcrypt.gensalt()).decode('utf-8'));" 2>&1)
 if [[ $? -ne 0 ]]; then
-	echo "Failed to generate Fleet password hash"
+    echo "Unable to add user to Fleet; Fleet Service account login failed"
-	exit 2
+    echo "$SALOGIN_OUTPUT"
+    exit 2
 fi
 
-MYSQL_OUTPUT=$(docker exec so-mysql mysql -u root --password=$MYSQL_PASS fleet -e \
+# Create New User
-    "INSERT INTO users (password,salt,username,email,admin,enabled) VALUES ('$FLEET_HASH','','$FLEET_USER','$FLEET_USER',1,1)" 2>&1)
+CREATE_OUTPUT=$(docker exec so-fleet fleetctl user create --email $USER_EMAIL --name $USER_EMAIL --password $USER_PASS --global-role admin 2>&1)
 
 if [[ $? -eq 0 ]]; then
     echo "Successfully added user to Fleet"
 else
     echo "Unable to add user to Fleet; user might already exist"
-    echo "$MYSQL_OUTPUT"
+    echo "$CREATE_OUTPUT"
     exit 2
-fi
+fi
+
+# Disable forced password reset
+MYSQL_OUTPUT=$(docker exec so-mysql mysql -u root --password=$MYSQL_PW fleet -e \
+"UPDATE users SET admin_forced_password_reset = 0 WHERE email = '$USER_EMAIL'" 2>&1)

salt/common/tools/sbin/so-fleet-user-delete

@@ -0,0 +1,56 @@
+#!/bin/bash
+#
+# Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. /usr/sbin/so-common
+
+usage() {
+    echo "Usage: $0 <user-email>"
+    echo ""
+    echo "Deletes a user in Fleet"
+    exit 1
+}
+
+if [ $# -ne 1 ]; then
+  usage
+fi
+
+USER_EMAIL=$1
+FLEET_SA_EMAIL=$(lookup_pillar_secret fleet_sa_email)
+FLEET_SA_PW=$(lookup_pillar_secret fleet_sa_password)
+
+# Config fleetctl & login with the SO Service Account
+CONFIG_OUTPUT=$(docker exec so-fleet fleetctl config set --address https://127.0.0.1:8080 --tls-skip-verify --url-prefix /fleet  2>&1 )
+SALOGIN_OUTPUT=$(docker exec so-fleet fleetctl login --email $FLEET_SA_EMAIL --password $FLEET_SA_PW 2>&1)
+
+if [[ $? -ne 0 ]]; then
+    echo "Unable to delete user from Fleet; Fleet Service account login failed"
+    echo "$SALOGIN_OUTPUT"
+    exit 2
+fi
+
+# Delete User
+DELETE_OUTPUT=$(docker exec so-fleet fleetctl user delete --email $USER_EMAIL 2>&1)
+
+if [[ $? -eq 0 ]]; then
+    echo "Successfully deleted user from Fleet"
+else
+    echo "Unable to delete user from Fleet"
+    echo "$DELETE_OUTPUT"
+    exit 2
+fi
+
+

salt/common/tools/sbin/so-fleet-user-update

@@ -36,9 +36,9 @@ FLEET_USER=$USER
 
 # test existence of user
 MYSQL_OUTPUT=$(docker exec so-mysql mysql -u root --password=$MYSQL_PASS fleet -e \
-    "SELECT count(1) FROM users WHERE username='$FLEET_USER'" 2>/dev/null | tail -1)
+    "SELECT count(1) FROM users WHERE email='$FLEET_USER'" 2>/dev/null | tail -1)
 if [[ $? -ne 0 ]] || [[ $MYSQL_OUTPUT -ne 1 ]] ; then
-    echo "Test for username [${FLEET_USER}] failed"
+    echo "Test for email [${FLEET_USER}] failed"
     echo "    expect 1 hit in users database, return $MYSQL_OUTPUT hit(s)."
     echo "Unable to update Fleet user password."
     exit 2
@@ -64,7 +64,7 @@ fi
 
 
 MYSQL_OUTPUT=$(docker exec so-mysql mysql -u root --password=$MYSQL_PASS fleet -e \
-    "UPDATE users SET password='$FLEET_HASH', salt='' where username='$FLEET_USER'" 2>&1)
+    "UPDATE users SET password='$FLEET_HASH', salt='' where email='$FLEET_USER'" 2>&1)
 
 if [[ $? -eq 0 ]]; then
     echo "Successfully updated Fleet user password"

salt/common/tools/sbin/so-import-evtx

@@ -25,6 +25,7 @@
 
 INDEX_DATE=$(date +'%Y.%m.%d')
 RUNID=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 8 | head -n 1)
+LOG_FILE=/nsm/import/evtx-import.log
 
 . /usr/sbin/so-common
 
@@ -41,14 +42,17 @@ function evtx2es() {
   EVTX=$1
   HASH=$2
   
+  ES_PW=$(lookup_pillar "auth:users:so_elastic_user:pass" "elasticsearch")
+  ES_USER=$(lookup_pillar "auth:users:so_elastic_user:user" "elasticsearch")
+
   docker run --rm \
     -v "$EVTX:/tmp/$RUNID.evtx" \
     --entrypoint evtx2es \
     {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-pcaptools:{{ VERSION }} \
     --host {{ MANAGERIP }} --scheme https \
     --index so-beats-$INDEX_DATE --pipeline import.wel \
-    --login {{ES_USER}} --pwd {{ES_PW}} \
+    --login $ES_USER --pwd $ES_PW \
-    "/tmp/$RUNID.evtx" 1>/dev/null 2>/dev/null
+    "/tmp/$RUNID.evtx" >> $LOG_FILE 2>&1
 
   docker run --rm \
     -v "$EVTX:/tmp/import.evtx" \

salt/common/tools/sbin/so-ip-update

@@ -8,9 +8,9 @@ fi
 
 echo "This tool will update a manager's IP address to the new IP assigned to the management network interface."
 
-echo
+echo ""
 echo "WARNING: This tool is still undergoing testing, use at your own risk!"
-echo
+echo ""
 
 if [ -z "$OLD_IP" ]; then
 	OLD_IP=$(lookup_pillar "managerip")
@@ -27,7 +27,7 @@ if [ -z "$NEW_IP" ]; then
 	NEW_IP=$(ip -4 addr list $iface | grep inet | cut -d' ' -f6 | cut -d/ -f1)
 
 	if [ -z "$NEW_IP" ]; then
-		fail "Unable to detect new IP on interface $iface.    "
+		fail "Unable to detect new IP on interface $iface."
 	fi
 
 	echo "Detected new IP $NEW_IP on interface $iface."
@@ -39,15 +39,20 @@ fi
 
 echo "About to change old IP $OLD_IP to new IP $NEW_IP."
 
-echo
+echo ""
 read -n 1 -p "Would you like to continue? (y/N) " CONTINUE
-echo
+echo ""
 
 if [ "$CONTINUE" == "y" ]; then
-  for file in $(grep -rlI $OLD_IP /opt/so/saltstack /etc); do
+	for file in $(grep -rlI $OLD_IP /opt/so/saltstack /etc); do
-  	echo "Updating file: $file"
+		echo "Updating file: $file"
-    sed -i "s|$OLD_IP|$NEW_IP|g" $file
+		sed -i "s|$OLD_IP|$NEW_IP|g" $file
-  done
+	done
+
+	echo "Granting MySQL root user permissions on $NEW_IP"
+	docker exec -i so-mysql mysql --user=root --password=$(lookup_pillar_secret 'mysql') -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'$NEW_IP' IDENTIFIED BY '$(lookup_pillar_secret 'mysql')' WITH GRANT OPTION;" &> /dev/null
+	echo "Removing MySQL root user from $OLD_IP"
+	docker exec -i so-mysql mysql --user=root --password=$(lookup_pillar_secret 'mysql') -e "DROP USER 'root'@'$OLD_IP';" &> /dev/null
 
 	echo "The IP has been changed from $OLD_IP to $NEW_IP."
 
@@ -60,4 +65,4 @@ if [ "$CONTINUE" == "y" ]; then
 	fi
 else
 	echo "Exiting without changes."
-fi
+fi

salt/common/tools/sbin/so-kibana-savedobjects-defaults

@@ -1,5 +1,5 @@
 #!/bin/bash
-#
+
 # Copyright 2014,2015,2016,2017,2018,2019,2020,2021 Security Onion Solutions, LLC
 #
 # This program is free software: you can redistribute it and/or modify
@@ -17,42 +17,14 @@
 
 . /usr/sbin/so-common
 
-usage() {
+echo $banner
-    echo "Usage: $0 <user-name>"
+echo "Running kibana.so_savedobjects_defaults Salt state to restore default saved objects."
-    echo ""
+printf "This could take a while if another Salt job is running. \nRun this command with --force to stop all Salt jobs before proceeding.\n"
-    echo "Enables or disables a user in Fleet"
+echo $banner
-    exit 1
-}
 
-if [ $# -ne 2 ]; then
+    if [ "$1" = "--force" ]; then
-  usage
+        printf "\nForce-stopping all Salt jobs before proceeding\n\n"
-fi
+        salt-call saltutil.kill_all_jobs
+    fi
 
-USER=$1
+salt-call state.apply kibana.so_savedobjects_defaults -linfo queue=True
-
-MYSQL_PASS=$(lookup_pillar_secret mysql)
-FLEET_IP=$(lookup_pillar fleet_ip)
-FLEET_USER=$USER
-
-case "${2^^}" in
-    FALSE | NO | 0)
-        FLEET_STATUS=0
-        ;;
-    TRUE | YES | 1)
-        FLEET_STATUS=1
-        ;;
-    *)
-        usage
-        ;;
-esac
-
-MYSQL_OUTPUT=$(docker exec so-mysql mysql -u root --password=$MYSQL_PASS fleet -e \
-    "UPDATE users SET enabled=$FLEET_STATUS WHERE username='$FLEET_USER'" 2>&1)
-
-if [[ $? -eq 0 ]]; then
-    echo "Successfully updated user in Fleet"
-else
-    echo "Failed to update user in Fleet"
-    echo $resp
-    exit 2
-fi

salt/common/tools/sbin/so-kibana-space-defaults

@@ -1,5 +1,5 @@
 . /usr/sbin/so-common
-
+{% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
 wait_for_web_response "http://localhost:5601/app/kibana" "Elastic" 300 "{{ ELASTICCURL }}"
 ## This hackery will be removed if using Elastic Auth ##
 
@@ -9,5 +9,9 @@ SESSIONCOOKIE=$({{ ELASTICCURL }} -c - -X GET http://localhost:5601/ | grep sid
 # Disable certain Features from showing up in the Kibana UI
 echo
 echo "Setting up default Space:"
+{% if HIGHLANDER %}
+{{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["enterpriseSearch"]} ' >> /opt/so/log/kibana/misc.log
+{% else %}
 {{ ELASTICCURL }} -b "sid=$SESSIONCOOKIE" -L -X PUT "localhost:5601/api/spaces/space/default" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d' {"id":"default","name":"Default","disabledFeatures":["ml","enterpriseSearch","siem","logs","infrastructure","apm","uptime","monitoring","stackAlerts","actions","fleet"]} ' >> /opt/so/log/kibana/misc.log
+{% endif %}
 echo

salt/common/tools/sbin/so-redis-count

@@ -17,4 +17,4 @@
 
 . /usr/sbin/so-common
 
-docker exec -it so-redis redis-cli llen logstash:unparsed
+docker exec so-redis redis-cli llen logstash:unparsed

salt/common/tools/sbin/so-rule

@@ -405,7 +405,7 @@ def main():
   enabled_list.set_defaults(func=list_enabled_rules)
   
 
-  search_term_help='A quoted regex search term (ex: "\$EXTERNAL_NET")'
+  search_term_help='A properly escaped regex search term (ex: "\\\$EXTERNAL_NET")'
   replace_term_help='The text to replace the search term with'
 
   # Modify actions

salt/common/tools/sbin/so-salt-minion-check

@@ -92,6 +92,10 @@ if [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+$UPTIME_REQ))  ]; then
     log "last highstate completed at `date -d @$LAST_HIGHSTATE_END`" I
     log "checking if any jobs are running" I
     logCmd "salt-call --local saltutil.running" I
+    log "ensure salt.minion-state-apply-test is enabled" I
+    logCmd "salt-call state.enable salt.minion-state-apply-test" I
+    log "ensure highstate is enabled" I
+    logCmd "salt-call state.enable highstate" I
     log "killing all salt-minion processes" I
     logCmd "pkill -9 -ef /usr/bin/salt-minion" I
     log "starting salt-minion service" I
@@ -101,4 +105,4 @@ if [ $CURRENT_TIME -ge $((SYSTEM_START_TIME+$UPTIME_REQ))  ]; then
   fi
 else
   log "system uptime only $((CURRENT_TIME-SYSTEM_START_TIME)) seconds does not meet $UPTIME_REQ second requirement." I
-fi
+fi

salt/common/tools/sbin/so-user

@@ -101,6 +101,9 @@ function validatePassword() {
   if [[ $len -lt 6 ]]; then
     fail "Password does not meet the minimum requirements"
   fi
+  if [[ $len -gt 72 ]]; then
+    fail "Password is too long (max: 72)"
+  fi
   check_password_and_exit "$password"
 }
 
@@ -179,6 +182,10 @@ function ensureRoleFileExists() {
       echo "Database file does not exist yet, installation is likely not yet complete."
     fi
 
+    if [[ -d "$socRolesFile" ]]; then
+      echo "Removing invalid roles directory created by Docker"
+      rm -fr "$socRolesFile"
+    fi
     mv "${rolesTmpFile}" "${socRolesFile}"
   fi
 }
@@ -237,8 +244,12 @@ function syncElastic() {
   if [[ -f "$databasePath" && -f "$socRolesFile" ]]; then
     # Append the SOC users 
     echo "select '{\"user\":\"' || ici.identifier || '\", \"data\":' || ic.config || '}'" \
-      "from identity_credential_identifiers ici, identity_credentials ic " \
+      "from identity_credential_identifiers ici, identity_credentials ic, identities i " \
-      "where ici.identity_credential_id=ic.id and instr(ic.config, 'hashed_password') " \
+      "where " \
+      "      ici.identity_credential_id=ic.id " \
+      "  and ic.identity_id=i.id " \
+      "  and instr(ic.config, 'hashed_password') " \
+      "  and i.state == 'active' " \
       "order by ici.identifier;" | \
       sqlite3 "$databasePath" | \
       jq -r '.user + ":" + .data.hashed_password' \
@@ -381,6 +392,19 @@ EOF
   fi
 }
 
+function migrateLockedUsers() {
+  # This is a migration function to convert locked users from prior to 2.3.90
+  # to inactive users using the newer Kratos functionality. This should only
+  # find locked users once.
+  lockedEmails=$(curl -s http://localhost:4434/identities | jq -r '.[] | select(.traits.status == "locked") | .traits.email')
+  if [[ -n "$lockedEmails" ]]; then
+    echo "Disabling locked users..."
+    for email in $lockedEmails; do
+      updateStatus "$email" locked
+    done
+  fi
+}
+
 function updateStatus() {
   email=$1
   status=$2
@@ -391,24 +415,18 @@ function updateStatus() {
   response=$(curl -Ss -L "${kratosUrl}/identities/$identityId")
   [[ $? != 0 ]] && fail "Unable to communicate with Kratos"
 
-  oldConfig=$(echo "select config from identity_credentials where identity_id='${identityId}';" | sqlite3 "$databasePath")
+  schemaId=$(echo "$response" | jq -r .schema_id)
+
+  # Capture traits and remove obsolete 'status' trait if exists
+  traitBlock=$(echo "$response" | jq -c .traits | sed -re 's/,?"status":".*?"//')
+
+  state="active"
   if [[ "$status" == "locked" ]]; then
-    config=$(echo $oldConfig | sed -e 's/hashed/locked/')
+    state="inactive"
-    echo "update identity_credentials set config=CAST('${config}' as BLOB) where identity_id='${identityId}';" | sqlite3 "$databasePath"
+  fi
-    [[ $? != 0 ]] && fail "Unable to lock credential record"
+  body="{ \"schema_id\": \"$schemaId\", \"state\": \"$state\", \"traits\": $traitBlock }"
-
+  response=$(curl -fSsL -XPUT "${kratosUrl}/identities/$identityId" -d "$body")
-    echo "delete from sessions where identity_id='${identityId}';" | sqlite3 "$databasePath"
+  [[ $? != 0 ]] && fail "Unable to update user"
-    [[ $? != 0 ]] && fail "Unable to invalidate sessions"    
-  else
-    config=$(echo $oldConfig | sed -e 's/locked/hashed/')
-    echo "update identity_credentials set config=CAST('${config}' as BLOB) where identity_id='${identityId}';" | sqlite3 "$databasePath"
-    [[ $? != 0 ]] && fail "Unable to unlock credential record"
-  fi  
-
-  updatedJson=$(echo "$response" | jq ".traits.status = \"$status\" | del(.verifiable_addresses) | del(.id) | del(.schema_url) | del(.created_at) | del(.updated_at)")
-  response=$(curl -Ss -XPUT -L ${kratosUrl}/identities/$identityId -d "$updatedJson")
-  [[ $? != 0 ]] && fail "Unable to mark user as locked"
-
 }
 
 function updateUser() {
@@ -431,7 +449,7 @@ function deleteUser() {
 
   rolesTmpFile="${socRolesFile}.tmp"
   createFile "$rolesTmpFile" "$soUID" "$soGID"
-  grep -v "$id" "$socRolesFile" > "$rolesTmpFile"
+  grep -v "$identityId" "$socRolesFile" > "$rolesTmpFile"
   mv "$rolesTmpFile" "$socRolesFile"
 }
 
@@ -499,7 +517,7 @@ case "${operation}" in
     syncAll
     echo "Successfully enabled user"
     check_container thehive && so-thehive-user-enable "$email" true
-    check_container fleet && so-fleet-user-enable "$email" true   
+    echo "Fleet user will need to be recreated manually with so-fleet-user-add"
     ;;
 
   "disable")
@@ -511,7 +529,7 @@ case "${operation}" in
     syncAll
     echo "Successfully disabled user"
     check_container thehive && so-thehive-user-enable "$email" false
-    check_container fleet && so-fleet-user-enable "$email" false   
+    check_container fleet && so-fleet-user-delete "$email"   
     ;;    
 
   "delete")
@@ -523,7 +541,7 @@ case "${operation}" in
     syncAll
     echo "Successfully deleted user"
     check_container thehive && so-thehive-user-enable "$email" false
-    check_container fleet && so-fleet-user-enable "$email" false
+    check_container fleet && so-fleet-user-delete "$email"
     ;;
 
   "sync")
@@ -547,6 +565,11 @@ case "${operation}" in
     echo "Password is acceptable"
     ;;
 
+  "migrate")
+    migrateLockedUsers
+    echo "User migration complete"
+    ;;
+
   *)
     fail "Unsupported operation: $operation"
     ;;

salt/common/tools/sbin/soup

@@ -221,6 +221,19 @@ check_local_mods() {
 
 # {% endraw %}
 
+check_pillar_items() {
+  local pillar_output=$(salt-call pillar.items --out=json)
+
+  cond=$(jq '.local | has("_errors")' <<< "$pillar_output")
+  if [[ "$cond" == "true" ]]; then
+    printf "\nThere is an issue rendering the manager's pillars. Please correct the issues in the sls files mentioned below before running SOUP again.\n\n"
+    jq '.local._errors[]' <<< "$pillar_output"
+    exit 0
+  else
+    printf "\nThe manager's pillars can be rendered. We can proceed with SOUP.\n\n"
+  fi
+}
+
 check_sudoers() {
   if grep -q "so-setup" /etc/sudoers; then
     echo "There is an entry for so-setup in the sudoers file, this can be safely deleted using \"visudo\"."
@@ -380,13 +393,11 @@ preupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Checking to see if changes are needed."
 
-    [[ "$INSTALLEDVERSION" =~ rc.1 ]] && rc1_to_rc2
+    [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_to_2.3.20
-    [[ "$INSTALLEDVERSION" =~ rc.2 ]] && rc2_to_rc3
+    [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_to_2.3.30
-    [[ "$INSTALLEDVERSION" =~ rc.3 ]] && rc3_to_2.3.0
+    [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_to_2.3.50
-    [[ "$INSTALLEDVERSION" == 2.3.0 || "$INSTALLEDVERSION" == 2.3.1 || "$INSTALLEDVERSION" == 2.3.2 || "$INSTALLEDVERSION" == 2.3.10 ]] && up_2.3.0_to_2.3.20
+    [[ "$INSTALLEDVERSION" == 2.3.50 || "$INSTALLEDVERSION" == 2.3.51 || "$INSTALLEDVERSION" == 2.3.52 || "$INSTALLEDVERSION" == 2.3.60 || "$INSTALLEDVERSION" == 2.3.61 || "$INSTALLEDVERSION" == 2.3.70 ]] && up_to_2.3.80
-    [[ "$INSTALLEDVERSION" == 2.3.20 || "$INSTALLEDVERSION" == 2.3.21 ]] && up_2.3.2X_to_2.3.30
+    [[ "$INSTALLEDVERSION" == 2.3.80 ]] && up_to_2.3.90
-    [[ "$INSTALLEDVERSION" == 2.3.30 || "$INSTALLEDVERSION" == 2.3.40 ]] && up_2.3.3X_to_2.3.50
-    [[ "$INSTALLEDVERSION" == 2.3.50 || "$INSTALLEDVERSION" == 2.3.51 || "$INSTALLEDVERSION" == 2.3.52 || "$INSTALLEDVERSION" == 2.3.60 || "$INSTALLEDVERSION" == 2.3.61 || "$INSTALLEDVERSION" == 2.3.70 ]] && up_2.3.5X_to_2.3.80
     true
 }
 
@@ -394,119 +405,66 @@ postupgrade_changes() {
     # This function is to add any new pillar items if needed.
     echo "Running post upgrade processes."
     
-    [[ "$POSTVERSION" =~ rc.1 ]] && post_rc1_to_rc2
+    [[ "$POSTVERSION" == 2.3.0 || "$POSTVERSION" == 2.3.1 || "$POSTVERSION" == 2.3.2 || "$POSTVERSION" == 2.3.10 || "$POSTVERSION" == 2.3.20 ]] && post_to_2.3.21
-    [[ "$POSTVERSION" == 2.3.20 || "$POSTVERSION" == 2.3.21 ]] && post_2.3.2X_to_2.3.30
+    [[ "$POSTVERSION" == 2.3.21 || "$POSTVERSION" == 2.3.30 ]] && post_to_2.3.40
-    [[ "$POSTVERSION" == 2.3.30 ]] && post_2.3.30_to_2.3.40
+    [[ "$POSTVERSION" == 2.3.40 || "$POSTVERSION" == 2.3.50 || "$POSTVERSION" == 2.3.51 || "$POSTVERSION" == 2.3.52 ]] && post_to_2.3.60
-    [[ "$POSTVERSION" == 2.3.50 ]] && post_2.3.5X_to_2.3.60
+    [[ "$POSTVERSION" == 2.3.60 || "$POSTVERSION" == 2.3.61 || "$POSTVERSION" == 2.3.70 || "$POSTVERSION" == 2.3.80 ]] && post_to_2.3.90
     true
 }
 
-post_rc1_to_2.3.21() {
+post_to_2.3.21() {
   salt-call state.apply playbook.OLD_db_init
   rm -f /opt/so/rules/elastalert/playbook/*.yaml
   so-playbook-ruleupdate >> /root/soup_playbook_rule_update.log 2>&1 &
   POSTVERSION=2.3.21
 }
 
-post_2.3.2X_to_2.3.30() {
+post_to_2.3.40() {
-  so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
-  POSTVERSION=2.3.30
-}
-
-post_2.3.30_to_2.3.40() {
   so-playbook-sigma-refresh >> /root/soup_playbook_sigma_refresh.log 2>&1 &
   so-kibana-space-defaults
   POSTVERSION=2.3.40
 }
 
-post_2.3.5X_to_2.3.60() {
+post_to_2.3.60() {
+  for table in identity_recovery_addresses selfservice_recovery_flows selfservice_registration_flows selfservice_verification_flows identities identity_verification_tokens identity_credentials selfservice_settings_flows identity_recovery_tokens continuity_containers identity_credential_identifiers identity_verifiable_addresses courier_messages selfservice_errors sessions selfservice_login_flows
+  do
+    echo "Forcing Kratos network migration: $table"
+    sqlite3 /opt/so/conf/kratos/db/db.sqlite "update $table set nid=(select id from networks limit 1);"
+  done
+
   POSTVERSION=2.3.60
 }
 
+post_to_2.3.90() {
+  # Do Kibana dashboard things
+  salt-call state.apply kibana.so_savedobjects_defaults queue=True
 
-rc1_to_rc2() {
+  # Create FleetDM service account
+    FLEET_MANAGER=$(lookup_pillar fleet_manager)
+    if [[ "$FLEET_MANAGER" == "True" ]]; then
+      FLEET_SA_EMAIL=$(lookup_pillar_secret fleet_sa_email)
+      FLEET_SA_PW=$(lookup_pillar_secret fleet_sa_password)
+      MYSQL_PW=$(lookup_pillar_secret mysql)
 
-  # Move the static file to global.sls
+      FLEET_HASH=$(docker exec so-soctopus python -c "import bcrypt; print(bcrypt.hashpw('$FLEET_SA_PW'.encode('utf-8'), bcrypt.gensalt()).decode('utf-8'));" 2>&1)
-  echo "Migrating static.sls to global.sls"
+      MYSQL_OUTPUT=$(docker exec so-mysql mysql -u root --password=$MYSQL_PW fleet -e \
-  mv -v /opt/so/saltstack/local/pillar/static.sls /opt/so/saltstack/local/pillar/global.sls >> "$SOUP_LOG" 2>&1
+      "INSERT INTO users (password,salt,email,name,global_role) VALUES ('$FLEET_HASH','','$FLEET_SA_EMAIL','$FLEET_SA_EMAIL','admin')" 2>&1)
-  sed -i '1c\global:' /opt/so/saltstack/local/pillar/global.sls >> "$SOUP_LOG" 2>&1
 
-  # Moving baseurl from minion sls file to inside global.sls
+      if [[ $? -eq 0 ]]; then
-  local line=$(grep '^  url_base:' /opt/so/saltstack/local/pillar/minions/$MINIONID.sls)
+          echo "Successfully added service account to Fleet"
-  sed -i '/^  url_base:/d' /opt/so/saltstack/local/pillar/minions/$MINIONID.sls;
+      else
-  sed -i "/^global:/a \\$line" /opt/so/saltstack/local/pillar/global.sls;
+          echo "Unable to add service account to Fleet"
+          echo "$MYSQL_OUTPUT"
+      fi
+    fi
 
-  # Adding play values to the global.sls
-  local HIVEPLAYSECRET=$(get_random_value)
-  local CORTEXPLAYSECRET=$(get_random_value)
-  sed -i "/^global:/a \\  hiveplaysecret: $HIVEPLAYSECRET" /opt/so/saltstack/local/pillar/global.sls;
-  sed -i "/^global:/a \\  cortexplaysecret: $CORTEXPLAYSECRET" /opt/so/saltstack/local/pillar/global.sls;
-
-  # Move storage nodes to hostname for SSL
-  # Get a list we can use:
-  grep -A1 searchnode /opt/so/saltstack/local/pillar/data/nodestab.sls | grep -v '\-\-' | sed '$!N;s/\n/ /' | awk '{print $1,$3}' | awk '/_searchnode:/{gsub(/\_searchnode:/, "_searchnode"); print}' >/tmp/nodes.txt
-  # Remove the nodes from cluster settings
-  while read p; do
-  local NAME=$(echo $p | awk '{print $1}')
-  local IP=$(echo $p | awk '{print $2}')
-  echo "Removing the old cross cluster config for $NAME"
-  curl -XPUT -H 'Content-Type: application/json' http://localhost:9200/_cluster/settings -d '{"persistent":{"cluster":{"remote":{"'$NAME'":{"skip_unavailable":null,"seeds":null}}}}}'
-  done </tmp/nodes.txt
-  # Add the nodes back using hostname
-  while read p; do
-    local NAME=$(echo $p | awk '{print $1}')
-    local EHOSTNAME=$(echo $p | awk -F"_" '{print $1}')
-    local IP=$(echo $p | awk '{print $2}')
-    echo "Adding the new cross cluster config for $NAME"
-    curl -XPUT http://localhost:9200/_cluster/settings -H'Content-Type: application/json' -d '{"persistent": {"search": {"remote": {"'$NAME'": {"skip_unavailable": "true", "seeds": ["'$EHOSTNAME':9300"]}}}}}'
-  done </tmp/nodes.txt
-
-  INSTALLEDVERSION=rc.2
-}
-
-rc2_to_rc3() {
-
-  # move location of local.rules
-  cp /opt/so/saltstack/default/salt/idstools/localrules/local.rules /opt/so/saltstack/local/salt/idstools/local.rules
   
-  if [ -f /opt/so/saltstack/local/salt/idstools/localrules/local.rules ]; then
-    cat /opt/so/saltstack/local/salt/idstools/localrules/local.rules >> /opt/so/saltstack/local/salt/idstools/local.rules
-  fi
-  rm -rf /opt/so/saltstack/local/salt/idstools/localrules
-  rm -rf /opt/so/saltstack/default/salt/idstools/localrules
-
-  # Rename mdengine to MDENGINE
-  sed -i "s/  zeekversion/  mdengine/g" /opt/so/saltstack/local/pillar/global.sls
-  # Enable Strelka Rules
-  sed -i "/  rules:/c\  rules: 1" /opt/so/saltstack/local/pillar/global.sls
-
-  INSTALLEDVERSION=rc.3
 
+  POSTVERSION=2.3.90
 }
 
-rc3_to_2.3.0() {
-  # Fix Tab Complete
-  if [ ! -f /etc/profile.d/securityonion.sh ]; then
-    echo "complete -cf sudo" > /etc/profile.d/securityonion.sh
-  fi
 
-  {
+up_to_2.3.20(){
-    echo "redis_settings:"
-    echo "  redis_maxmemory: 827"
-    echo "playbook:"
-    echo "  api_key: de6639318502476f2fa5aa06f43f51fb389a3d7f" 
-  } >> /opt/so/saltstack/local/pillar/global.sls
-
-  sed -i 's/playbook:/playbook_db:/' /opt/so/saltstack/local/pillar/secrets.sls
-  {
-    echo "playbook_admin: $(get_random_value)"
-    echo "playbook_automation: $(get_random_value)"
-  } >> /opt/so/saltstack/local/pillar/secrets.sls
-
-  INSTALLEDVERSION=2.3.0
-}
-
-up_2.3.0_to_2.3.20(){
   DOCKERSTUFFBIP=$(echo $DOCKERSTUFF | awk -F'.' '{print $1,$2,$3,1}' OFS='.')/24
   # Remove PCAP from global
   sed '/pcap:/d' /opt/so/saltstack/local/pillar/global.sls
@@ -544,7 +502,7 @@ up_2.3.0_to_2.3.20(){
   INSTALLEDVERSION=2.3.20
 }
 
-up_2.3.2X_to_2.3.30() {
+up_to_2.3.30() {
   # Replace any curly brace scalars with the same scalar in single quotes
   readarray -t minion_pillars <<< "$(find /opt/so/saltstack/local/pillar/minions -type f -name '*.sls')"
   for pillar in "${minion_pillars[@]}"; do
@@ -567,32 +525,7 @@ up_2.3.2X_to_2.3.30() {
   INSTALLEDVERSION=2.3.30
 }
 
-upgrade_to_2.3.50_repo() {
+up_to_2.3.50() {
-  echo "Performing repo changes."
-  if [[ "$OS" == "centos" ]]; then
-    # Import GPG Keys
-    gpg_rpm_import
-    echo "Disabling fastestmirror."
-    disable_fastestmirror
-    echo "Deleting unneeded repo files."
-    DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh')
-
-    for DELREPO in "${DELREPOS[@]}"; do
-      if [[ -f "/etc/yum.repos.d/$DELREPO.repo" ]]; then
-        echo "Deleting $DELREPO.repo"
-        rm -f "/etc/yum.repos.d/$DELREPO.repo"
-      fi
-    done
-    if [[ $is_airgap -eq 1 ]]; then
-      # Copy the new repo file if not airgap
-      cp $UPDATE_DIR/salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/
-      yum clean all
-      yum repolist
-    fi
-  fi
-}
-
-up_2.3.3X_to_2.3.50() {
   
   cat <<EOF > /tmp/supersed.txt
 /so-zeek:/ {
@@ -624,7 +557,7 @@ EOF
   INSTALLEDVERSION=2.3.50
 }
 
-up_2.3.5X_to_2.3.80() {
+up_to_2.3.80() {
   
   # Remove watermark settings from global.sls
   sed -i '/  cluster_routing_allocation_disk/d' /opt/so/saltstack/local/pillar/global.sls
@@ -664,6 +597,51 @@ up_2.3.5X_to_2.3.80() {
   INSTALLEDVERSION=2.3.80
 }
 
+up_to_2.3.90() {
+  for i in manager managersearch eval standalone; do
+    if compgen -G "/opt/so/saltstack/local/pillar/minions/*_$i.sls" > /dev/null; then
+      echo "soc:" >> /opt/so/saltstack/local/pillar/minions/*_$i.sls
+      sed -i "/^soc:/a \\  es_index_patterns: '*:so-*,*:endgame-*'" /opt/so/saltstack/local/pillar/minions/*_$i.sls
+    fi
+  done
+  
+  # Create Endgame Hostgroup
+  so-firewall addhostgroup endgame
+  
+  # Force influx to generate a new cert
+  mv /etc/pki/influxdb.crt /etc/pki/influxdb.crt.2390upgrade
+  mv /etc/pki/influxdb.key /etc/pki/influxdb.key.2390upgrade
+  
+  # remove old common ingest pipeline in default
+  rm -vf /opt/so/saltstack/default/salt/elasticsearch/files/ingest/common
+  # if custom common, move from local ingest to local ingest-dynamic
+  mkdir -vp /opt/so/saltstack/local/salt/elasticsearch/files/ingest-dynamic
+  if [[ -f "/opt/so/saltstack/local/salt/elasticsearch/files/ingest/common" ]]; then
+    mv -v /opt/so/saltstack/local/salt/elasticsearch/files/ingest/common /opt/so/saltstack/local/salt/elasticsearch/files/ingest-dynamic/common
+    # since json file, we need to wrap with raw
+    sed -i '1s/^/{{'{% raw %}'}}\n/' /opt/so/saltstack/local/salt/elasticsearch/files/ingest-dynamic/common
+    sed -i -e '$a{{'{% endraw %}'}}\n' /opt/so/saltstack/local/salt/elasticsearch/files/ingest-dynamic/common
+  fi
+
+  # Generate FleetDM Service Account creds if they do not exist
+  if grep -q "fleet_sa_email" /opt/so/saltstack/local/pillar/secrets.sls; then
+    echo "FleetDM Service Account credentials already created..."
+  else
+    echo "Generating FleetDM Service Account credentials..."
+    FLEETSAPASS=$(get_random_value)
+    	printf '%s\n'\
+        "  fleet_sa_email: service.account@securityonion.invalid"\
+        "  fleet_sa_password: $FLEETSAPASS"\
+        >> /opt/so/saltstack/local/pillar/secrets.sls
+
+  fi
+
+  sed -i -re 's/^(playbook_admin.*|playbook_automation.*)/  \1/g' /opt/so/saltstack/local/pillar/secrets.sls
+  
+  INSTALLEDVERSION=2.3.90
+}
+
+
 verify_upgradespace() {
   CURRENTSPACE=$(df -BG / | grep -v Avail | awk '{print $4}' | sed 's/.$//')
   if [ "$CURRENTSPACE" -lt "10" ]; then
@@ -805,17 +783,48 @@ upgrade_salt() {
   fi
 }
 
+upgrade_to_2.3.50_repo() {
+  echo "Performing repo changes."
+  if [[ "$OS" == "centos" ]]; then
+    # Import GPG Keys
+    gpg_rpm_import
+    echo "Disabling fastestmirror."
+    disable_fastestmirror
+    echo "Deleting unneeded repo files."
+    DELREPOS=('CentOS-Base' 'CentOS-CR' 'CentOS-Debuginfo' 'docker-ce' 'CentOS-fasttrack' 'CentOS-Media' 'CentOS-Sources' 'CentOS-Vault' 'CentOS-x86_64-kernel' 'epel' 'epel-testing' 'saltstack' 'wazuh')
+
+    for DELREPO in "${DELREPOS[@]}"; do
+      if [[ -f "/etc/yum.repos.d/$DELREPO.repo" ]]; then
+        echo "Deleting $DELREPO.repo"
+        rm -f "/etc/yum.repos.d/$DELREPO.repo"
+      fi
+    done
+    if [[ $is_airgap -eq 1 ]]; then
+      # Copy the new repo file if not airgap
+      cp $UPDATE_DIR/salt/repo/client/files/centos/securityonion.repo /etc/yum.repos.d/
+      yum clean all
+      yum repolist
+    fi
+  fi
+}
+
 verify_latest_update_script() {
+  #we need to render soup and so-common first since they contain jinja
+  salt-call slsutil.renderer $UPDATE_DIR/salt/common/tools/sbin/soup default_renderer='jinja' --local --out=newline_values_only --out-indent=-4 --out-file=/tmp/soup
+  sed -i -e '$a\' /tmp/soup
+  salt-call slsutil.renderer $UPDATE_DIR/salt/common/tools/sbin/so-common default_renderer='jinja' --local --out=newline_values_only --out-indent=-4 --out-file=/tmp/so-common
+  sed -i -e '$a\' /tmp/so-common
   # Check to see if the update scripts match. If not run the new one.
-  CURRENTSOUP=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/soup | awk '{print $1}')
+  CURRENTSOUP=$(md5sum /usr/sbin/soup | awk '{print $1}')
-  GITSOUP=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/soup | awk '{print $1}')
+  GITSOUP=$(md5sum /tmp/soup | awk '{print $1}')
-  CURRENTCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-common | awk '{print $1}')
+  CURRENTCMN=$(md5sum /usr/sbin/so-common | awk '{print $1}')
-  GITCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-common | awk '{print $1}')
+  GITCMN=$(md5sum /tmp/so-common | awk '{print $1}')
-  CURRENTIMGCMN=$(md5sum /opt/so/saltstack/default/salt/common/tools/sbin/so-image-common | awk '{print $1}')
+  CURRENTIMGCMN=$(md5sum /usr/sbin/so-image-common | awk '{print $1}')
   GITIMGCMN=$(md5sum $UPDATE_DIR/salt/common/tools/sbin/so-image-common | awk '{print $1}')
 
   if [[ "$CURRENTSOUP" == "$GITSOUP" && "$CURRENTCMN" == "$GITCMN" && "$CURRENTIMGCMN" == "$GITIMGCMN" ]]; then
     echo "This version of the soup script is up to date. Proceeding."
+    rm -f /tmp/soup /tmp/so-common
   else
     echo "You are not running the latest soup version. Updating soup and its components. Might take multiple runs to complete"
     cp $UPDATE_DIR/salt/common/tools/sbin/soup $DEFAULT_SALT_DIR/salt/common/tools/sbin/
@@ -831,6 +840,8 @@ verify_latest_update_script() {
 main() {
   trap 'check_err $?' EXIT
   
+  check_pillar_items
+
   echo "Checking to see if this is an airgap install."
   echo ""
   check_airgap
@@ -1037,6 +1048,9 @@ main() {
     echo "Checking sudoers file."
     check_sudoers
 
+    echo "Checking for necessary user migrations."
+    so-user migrate
+
     if [[ -n $lsl_msg ]]; then
       case $lsl_msg in
         'distributed')
@@ -1138,4 +1152,3 @@ fi
 
 echo "### Preparing soup at $(date) ###"
 main "$@" | tee -a $SOUP_LOG
-  

salt/curator/files/action/so-endgame-close.yml

@@ -0,0 +1,29 @@
+{%- set cur_close_days = salt['pillar.get']('elasticsearch:index_settings:so-endgame:close', 30) -%}
+---
+# Remember, leave a key empty if there is no value.  None will be a string,
+# not a Python "NoneType"
+#
+# Also remember that all examples have 'disable_action' set to True.  If you
+# want to use this action as a template, be sure to set this to False after
+# copying it.
+actions:
+  1:
+    action: close
+    description: >-
+      Close Endgame indices older than {{cur_close_days}} days.
+    options:
+      delete_aliases: False
+      timeout_override:
+      continue_if_exception: False
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: regex 
+      value: '^(logstash-endgame.*|so-endgame.*|endgame.*)$'
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{cur_close_days}}
+      exclude:

salt/curator/files/action/so-endgame-delete.yml

@@ -0,0 +1,27 @@
+{%- set DELETE_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-endgame:delete', 365) -%}
+---
+# Remember, leave a key empty if there is no value.  None will be a string,
+# not a Python "NoneType"
+#
+# Also remember that all examples have 'disable_action' set to True.  If you
+# want to use this action as a template, be sure to set this to False after
+# copying it.
+actions:
+  1:
+    action: delete_indices
+    description: >-
+      Delete Endgame indices when older than {{ DELETE_DAYS }} days.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: regex
+      value: '^(logstash-endgame.*|so-endgame.*|endgame.*)$'
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ DELETE_DAYS }}
+      exclude:

salt/curator/files/action/so-endgame-warm.yml

@@ -0,0 +1,23 @@
+{%- set WARM_DAYS = salt['pillar.get']('elasticsearch:index_settings:so-endgame:warm', 7) -%}
+actions:
+  1:
+    action: allocation
+    description: "Apply shard allocation filtering rules to the specified indices"
+    options:
+      key: box_type
+      value: warm
+      allocation_type: require
+      wait_for_completion: true
+      timeout_override:
+      continue_if_exception: false
+      disable_action: false
+    filters:
+    - filtertype: pattern
+      kind: regex
+      value: '^(logstash-endgame.*|so-endgame.*|endgame.*)$'
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: {{ WARM_DAYS }} 

salt/domainstats/init.sls

@@ -45,14 +45,15 @@ so-domainstatsimage:
 
 so-domainstats:
   docker_container.running:
-    - require:
-      - so-domainstatsimage
     - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-domainstats:{{ VERSION }}
     - hostname: domainstats
     - name: so-domainstats
     - user: domainstats
     - binds:
       - /opt/so/log/domainstats:/var/log/domain_stats
+    - require:
+      - file: dstatslogdir
+      - cmd: so-domainstatsimage
 
 append_so-domainstats_so-status.conf:
   file.append:
@@ -65,4 +66,4 @@ append_so-domainstats_so-status.conf:
   test.fail_without_changes:
     - name: {{sls}}_state_not_allowed
 
-{% endif %}
+{% endif %}

salt/elastalert/init.sls

@@ -111,17 +111,21 @@ so-elastalert:
     - image: {{ MANAGER }}:5000/{{ IMAGEREPO }}/so-elastalert:{{ VERSION }}
     - hostname: elastalert
     - name: so-elastalert
-    - user: elastalert
+    - user: so-elastalert
     - detach: True
     - binds:
       - /opt/so/rules/elastalert:/opt/elastalert/rules/:ro
       - /opt/so/log/elastalert:/var/log/elastalert:rw
       - /opt/so/conf/elastalert/modules/:/opt/elastalert/modules/:ro
-      - /opt/so/conf/elastalert/elastalert_config.yaml:/opt/config/elastalert_config.yaml:ro
+      - /opt/so/conf/elastalert/elastalert_config.yaml:/opt/elastalert/config.yaml:ro
     - extra_hosts:
       - {{MANAGER_URL}}:{{MANAGER_IP}}
     - require:
       - cmd: wait_for_elasticsearch
+      - file: elastarules
+      - file: elastalogdir
+      - file: elastacustmodulesdir
+      - file: elastaconf
     - watch:
       - file: elastaconf
 

salt/elasticsearch/auth.sls

@@ -1,8 +1,12 @@
-{% set so_elastic_user_pass = salt['random.get_str'](20) %}
+{% from 'allowed_states.map.jinja' import allowed_states %}
-{% set so_kibana_user_pass = salt['random.get_str'](20) %}
+{% if sls in allowed_states %}
-{% set so_logstash_user_pass = salt['random.get_str'](20) %}
+
-{% set so_beats_user_pass = salt['random.get_str'](20) %}
+  {% set so_elastic_user_pass = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', salt['random.get_str'](72)) %}
-{% set so_monitor_user_pass = salt['random.get_str'](20) %}
+  {% set so_kibana_user_pass = salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:pass', salt['random.get_str'](72)) %}
+  {% set so_logstash_user_pass = salt['pillar.get']('elasticsearch:auth:users:so_logstash_user:pass', salt['random.get_str'](72)) %}
+  {% set so_beats_user_pass = salt['pillar.get']('elasticsearch:auth:users:so_beats_user:pass', salt['random.get_str'](72)) %}
+  {% set so_monitor_user_pass = salt['pillar.get']('elasticsearch:auth:users:so_monitor_user:pass', salt['random.get_str'](72)) %}
+  {% set auth_enabled = salt['pillar.get']('elasticsearch:auth:enabled', False) %}
 
 elastic_auth_pillar:
   file.managed:
@@ -12,7 +16,7 @@ elastic_auth_pillar:
     - contents: |
         elasticsearch:
           auth:
-            enabled: False
+            enabled: {{ auth_enabled }}
             users:
               so_elastic_user:
                 user: so_elastic
@@ -29,11 +33,11 @@ elastic_auth_pillar:
               so_monitor_user:
                 user: so_monitor
                 pass: {{ so_monitor_user_pass }}
-    # since we are generating a random password, and we don't want that to happen everytime
+    - show_changes: False
-    # a highstate runs, we only manage the file each user isn't present in the file. if the
+{% else %}
-    # pillar file doesn't exists, then the default vault provided to pillar.get should not
+
-    # be within the file either, so it should then be created
+{{sls}}_state_not_allowed:
-    - unless:
+  test.fail_without_changes:
-    {% for so_app_user, values in salt['pillar.get']('elasticsearch:auth:users', {'so_noapp_user': {'user': 'r@NDumu53Rd0NtDOoP'}}).items() %}
+    - name: {{sls}}_state_not_allowed
-      - grep {{ values.user }} /opt/so/saltstack/local/pillar/elasticsearch/auth.sls
+
-    {% endfor%}
+{% endif %}

salt/elasticsearch/config.map.jinja

@@ -1,4 +1,5 @@
 {% import_yaml 'elasticsearch/defaults.yaml' as ESCONFIG with context %}
+{% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %}
 
 {% if not salt['pillar.get']('elasticsearch:auth:enabled', False) %}
   {% do ESCONFIG.elasticsearch.config.xpack.security.authc.anonymous.update({'username': 'anonymous_user', 'roles': 'superuser', 'authz_exception': 'true'}) %}
@@ -8,6 +9,9 @@
     {% if grains.id.split('_') | last in ['manager','managersearch'] %}
     {% if salt['pillar.get']('nodestab', {}) %}
       {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['master', 'data', 'remote_cluster_client']}) %}
+      {% if HIGHLANDER %}
+          {% do ESCONFIG.elasticsearch.config.node.roles.append('ml') %}
+      {% endif %}
       {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %}
       {% for SN, SNDATA in salt['pillar.get']('nodestab', {}).items() %}
         {% do ESCONFIG.elasticsearch.config.discovery.seed_hosts.append(SN.split('_')|first) %}
@@ -18,9 +22,15 @@
     {% endif %}
   {% else %}
     {% do ESCONFIG.elasticsearch.config.node.update({'roles': ['data', 'ingest']}) %}
+    {% if HIGHLANDER %}
+          {% do ESCONFIG.elasticsearch.config.node.roles.extend(['ml', 'master']) %}
+    {% endif %}
     {% do ESCONFIG.elasticsearch.config.node.attr.update({'box_type': 'hot'}) %}
     {% do ESCONFIG.elasticsearch.config.update({'discovery': {'seed_hosts': [grains.master]}}) %}
   {% endif %}
+    {% if HIGHLANDER %}
+        {% do ESCONFIG.elasticsearch.config.xpack.ml.update({'enabled': true}) %}
+    {% endif %}
 {% endif %}
 
 {% set ESCONFIG = salt['pillar.get']('elasticsearch:config', default=ESCONFIG.elasticsearch.config, merge=True) %}

salt/elasticsearch/files/ingest-dynamic/common

@@ -1,3 +1,5 @@
+{%- set HIGHLANDER = salt['pillar.get']('global:highlander', False) -%}
+{%- raw -%}
 {
   "description" : "common",
   "processors" : [
@@ -21,6 +23,26 @@
 		"properties": ["ip", "country_iso_code", "country_name", "continent_name", "region_iso_code", "region_name", "city_name", "timezone", "location"]
 	}
     },
+    {
+        "geoip":  {
+                "field": "destination.ip",
+                "target_field": "destination_geo",
+                "database_file": "GeoLite2-ASN.mmdb",
+                "ignore_missing": true,
+                "ignore_failure": true,
+                "properties": ["ip", "asn", "organization_name", "network"]
+        }
+    },
+    {
+        "geoip":  {
+                "field": "source.ip",
+                "target_field": "source_geo",
+                "database_file": "GeoLite2-ASN.mmdb",
+                "ignore_missing": true,
+                "ignore_failure": true,
+                "properties": ["ip", "asn", "organization_name", "network"]
+        }
+    },
     { "set":             { "if": "ctx.event?.severity == 1",   "field": "event.severity_label", "value": "low", "override": true }  },
     { "set":             { "if": "ctx.event?.severity == 2",   "field": "event.severity_label", "value": "medium", "override": true }  },
     { "set":             { "if": "ctx.event?.severity == 3",   "field": "event.severity_label", "value": "high", "override": true }  },
@@ -45,5 +67,16 @@
                 "index_name_format": "yyyy.MM.dd"
         }
     }
+{%- endraw %}
+{%- if HIGHLANDER %}
+    ,
+    {
+      "pipeline": {
+        "name": "ecs"
+      }
+    }
+{%- endif %}
+{%- raw %} 
   ]
 }
+{% endraw %}

salt/elasticsearch/files/ingest/beats.common

@@ -2,7 +2,7 @@
   "description" : "beats.common",
   "processors" : [
     { "pipeline":      { "if": "ctx.winlog?.channel == 'Microsoft-Windows-Sysmon/Operational'",   "name": "sysmon"  }  },
-    { "pipeline":      { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational'",  "name":"win.eventlogs" }  },
+    { "pipeline":      { "if": "ctx.winlog?.channel != 'Microsoft-Windows-Sysmon/Operational' && ctx.containsKey('winlog')",  "name":"win.eventlogs" }  },
     { "pipeline":    { "name": "common" } }
   ]
 }

salt/elasticsearch/files/ingest/ecs

@@ -0,0 +1,155 @@
+{
+  "description" : "ECS Testing Pipeline",
+  "processors": [
+    {
+      "append": {
+        "field": "event.category",
+        "value": [
+          "process"
+        ],
+        "if": "ctx?.wazuh?.data?.type == 'process'",
+        "tag": "test",
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "event.type",
+        "value": [
+          "start"
+        ],
+        "if": "ctx?.wazuh?.data?.type == 'process'",
+        "tag": "test",
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "event.type",
+        "value": "end",
+        "if": "ctx?.wazuh?.data?.type == 'process_end'",
+        "tag": "test",
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "user.name",
+        "copy_from": "process.user",
+        "ignore_empty_value": true,
+        "tag": "test",
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "host.os.type",
+        "copy_from": "wazuh.data.os.sysname",
+        "ignore_empty_value": true,
+        "tag": "test",
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "host.os.platform",
+        "copy_from": "wazuh.data.os.platform",
+        "ignore_empty_value": true,
+        "tag": "test",
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "host.os.name",
+        "copy_from": "wazuh.data.os.name",
+        "ignore_empty_value": true,
+        "tag": "test",
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "host.os.version",
+        "copy_from": "wazuh.data.os.version",
+        "ignore_empty_value": true,
+        "tag": "test",
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "signal.rule.name",
+        "copy_from": "rule.name",
+        "ignore_empty_value": true,
+        "tag": "test",
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "signal.rule.type",
+        "copy_from": "rule.category",
+        "ignore_empty_value": true,
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "signal.rule.threat.tactic.name",
+        "copy_from": "rule.mitre.tactic",
+        "ignore_empty_value": true,
+        "tag": "test",
+        "ignore_failure": true
+      }
+    },
+    {
+      "append": {
+        "field": "event.category",
+        "value": [
+          "authentication"
+        ],
+        "if": "if(ctx?.rule?.groups != null) {\n    if(ctx?.rule?.groups?.contains('authentication_success')) {\n        return true\n    }\n    if(ctx?.rule?.groups?.contains('authentication_failed')) {\n        return true\n    }\n    return false\n}",
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "event.outcome",
+        "value": "success",
+        "ignore_empty_value": true,
+        "if": "ctx?.rule?.groups != null && ctx?.rule?.groups.contains('authentication_success')",
+        "tag": "test",
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "event.outcome",
+        "value": "failure",
+        "ignore_empty_value": true,
+        "if": "ctx?.rule?.groups != null && ctx?.rule?.groups.contains('authentication_failed')",
+        "tag": "test",
+        "ignore_failure": true
+      }
+    },
+    {
+      "set": {
+        "field": "url.path",
+        "ignore_empty_value": true,
+        "tag": "test",
+        "ignore_failure": true,
+        "copy_from": "url.original"
+      }
+    },
+    {
+      "set": {
+        "field": "url.domain",
+        "ignore_empty_value": true,
+        "tag": "test",
+        "ignore_failure": true,
+        "copy_from": "kibana.log.meta.req.headers.origin"
+      }
+    }
+  ]
+}

salt/elasticsearch/init.sls

@@ -15,7 +15,8 @@
 {% from 'allowed_states.map.jinja' import allowed_states %}
 {% if sls in allowed_states %}
 
-
+include:
+  - ssl
 
 {% set VERSION = salt['pillar.get']('global:soversion', 'HH1.2.2') %}
 {% set IMAGEREPO = salt['pillar.get']('global:imagerepo') %}
@@ -130,6 +131,14 @@ esrolesdir:
     - group: 939
     - makedirs: True
 
+esingestdynamicconf:
+  file.recurse:
+    - name: /opt/so/conf/elasticsearch/ingest
+    - source: salt://elasticsearch/files/ingest-dynamic
+    - user: 930
+    - group: 939
+    - template: jinja
+
 esingestconf:
   file.recurse:
     - name: /opt/so/conf/elasticsearch/ingest
@@ -279,7 +288,26 @@ so-elasticsearch:
       - file: cacertz
       - file: esyml
       - file: esingestconf
+      - file: esingestdynamicconf
       - file: so-elasticsearch-pipelines-file
+    - require:
+      - file: esyml
+      - file: eslog4jfile
+      - file: nsmesdir
+      - file: eslogdir
+      - file: cacertz
+      - x509: /etc/pki/elasticsearch.crt
+      - x509: /etc/pki/elasticsearch.key
+      - file: elasticp12perms
+      {% if ismanager %}
+      - x509: pki_public_ca_crt
+      {% else %}
+      - x509: trusttheca
+      {% endif %}
+      {% if salt['pillar.get']('elasticsearch:auth:enabled', False) %}
+      - cmd: auth_users_roles_inode
+      - cmd: auth_users_inode
+      {% endif %}
 
 append_so-elasticsearch_so-status.conf:
   file.append:
@@ -302,6 +330,7 @@ so-elasticsearch-pipelines:
    - name: /opt/so/conf/elasticsearch/so-elasticsearch-pipelines {{ esclustername }}
    - onchanges:
       - file: esingestconf
+      - file: esingestdynamicconf
       - file: esyml
       - file: so-elasticsearch-pipelines-file
 

salt/elasticsearch/templates/so/so-common-template.json.jinja

@@ -1,12 +1,14 @@
 {%- set INDEX_SORTING = salt['pillar.get']('elasticsearch:index_sorting', True) %}
+{%- set REPLICAS = salt['pillar.get']('elasticsearch:replicas', 0) %}
+{%- set REFRESH = salt['pillar.get']('elasticsearch:index_settings:so-common:refresh', '30s') %}
 {
   "index_patterns": ["so-*"],
   "version":50001,
   "order":10,
   "settings":{
-    "number_of_replicas":0,
+    "number_of_replicas":{{ REPLICAS }},
     "number_of_shards":1,
-    "index.refresh_interval":"30s",
+    "index.refresh_interval":"{{ REFRESH }}",
     "index.routing.allocation.require.box_type":"hot",
     "index.mapping.total_fields.limit": "1500",
 {%- if INDEX_SORTING is sameas true %}

salt/filebeat/init.sls

@@ -25,9 +25,10 @@
 {% from 'filebeat/map.jinja' import SO with context %}
 {% set ES_INCLUDED_NODES = ['so-eval', 'so-standalone', 'so-managersearch', 'so-node', 'so-heavynode', 'so-import'] %}
 
+include:
+  - ssl
 #only include elastic state for certain nodes
 {% if grains.role in ES_INCLUDED_NODES %}
-include:
   - elasticsearch
 {% endif %}
 
@@ -66,7 +67,7 @@ fileregistrydir:
     - makedirs: True
 
 # This needs to be owned by root
-filebeatconfsync:
+filebeatconf:
   file.managed:
     - name: /opt/so/conf/filebeat/etc/filebeat.yml
     - source: salt://filebeat/etc/filebeat.yml
@@ -76,9 +77,10 @@ filebeatconfsync:
     - defaults:
         INPUTS: {{ salt['pillar.get']('filebeat:config:inputs', {}) }}
         OUTPUT: {{ salt['pillar.get']('filebeat:config:output', {}) }}
+    - show_changes: False
 
 # Filebeat module config file
-filebeatmoduleconfsync:
+filebeatmoduleconf:
   file.managed:
     - name: /opt/so/conf/filebeat/etc/module-setup.yml
     - source: salt://filebeat/etc/module-setup.yml
@@ -86,6 +88,7 @@ filebeatmoduleconfsync:
     - group: root
     - mode: 640
     - template: jinja
+    - show_changes: False
 
 sodefaults_module_conf:
   file.managed:
@@ -135,14 +138,21 @@ so-filebeat:
   {% endfor %}
 {% endfor %}
     - watch:
-      - file: /opt/so/conf/filebeat/etc/filebeat.yml
+      - file: filebeatconf
+    - require:
+      - file: filebeatconf
+      - file: filebeatmoduleconf
+      - file: filebeatmoduledir
+      - x509: conf_filebeat_crt
+      - x509: conf_filebeat_key
+      - x509: trusttheca
 
 {% if grains.role in ES_INCLUDED_NODES %}
 run_module_setup:
   cmd.run:
     - name: /usr/sbin/so-filebeat-module-setup
     - require:
-      - file: filebeatmoduleconfsync
+      - file: filebeatmoduleconf
       - docker_container: so-filebeat
     - onchanges:
       - docker_container: so-elasticsearch

salt/filebeat/thirdpartydefaults.yaml

@@ -244,6 +244,23 @@ third_party_filebeat:
         var.input: udp
         var.syslog_host: 0.0.0.0
         var.syslog_port: 9501
+    threatintel:
+      abuseurl:
+        enabled: false
+      abusemalware:
+        enabled: false
+      misp:
+        enabled: false
+      malwarebazaar:
+        enabled: false
+      otx:
+        enabled: false
+      anomali:
+        enabled: false
+      anomalithreatstream:
+        enabled: false
+      recordedfuture:
+        enabled: false
     zscaler:
       zia:
         enabled: false 

salt/firewall/assigned_hostgroups.map.yaml

@@ -162,6 +162,9 @@ role:
           elasticsearch_rest:
             portgroups:
               - {{ portgroups.elasticsearch_rest }}
+          endgame:
+            portgroups:
+              - {{ portgroups.endgame }}
           osquery_endpoint:
             portgroups:
               - {{ portgroups.fleet_api }}
@@ -248,6 +251,9 @@ role:
           elasticsearch_rest:
             portgroups:
               - {{ portgroups.elasticsearch_rest }}
+          endgame:
+            portgroups:
+              - {{ portgroups.endgame }}
           osquery_endpoint:
             portgroups:
               - {{ portgroups.fleet_api }}
@@ -337,6 +343,9 @@ role:
           elasticsearch_rest:
             portgroups:
               - {{ portgroups.elasticsearch_rest }}
+          endgame:
+            portgroups:
+              - {{ portgroups.endgame }}
           osquery_endpoint:
             portgroups:
               - {{ portgroups.fleet_api }}
@@ -594,4 +603,4 @@ role:
               - {{ portgroups.all }}
           minion:
             portgroups:
-              - {{ portgroups.salt_manager }}
+              - {{ portgroups.salt_manager }}

salt/firewall/portgroups.yaml

@@ -39,6 +39,9 @@ firewall:
       elasticsearch_rest:
         tcp:
           - 9200
+      endgame:
+        tcp:
+          - 3765
       fleet_api:
         tcp:
           - 8090

salt/fleet/event_update-enroll-secret.sls

@@ -1,4 +1,4 @@
-{% set ENROLLSECRET = salt['cmd.run']('docker exec so-fleet fleetctl get enroll-secret default') %}
+{% set ENROLLSECRET = salt['cmd.shell']('docker exec so-fleet fleetctl get enroll-secret --json | jq -r ".spec.secrets[].secret"') %}
 
 so/fleet:
   event.send:

salt/fleet/files/packs/osquery-config.conf

@@ -1,31 +1,34 @@
+---
 apiVersion: v1
-kind: options
+kind: config
 spec:
-  config:
+  agent_options:
-    decorators:
+    config:
-      always:
+      decorators:
-      - SELECT codename FROM os_version;
+        always:
-      - SELECT uuid AS live_query FROM system_info;
+        - SELECT codename FROM os_version;
-      - SELECT address AS endpoint_ip1 FROM interface_addresses where address not
+        - SELECT uuid AS live_query FROM system_info;
-        like '%:%' and address not like '127%' and address not like '169%' order by
+        - SELECT address AS endpoint_ip1 FROM interface_addresses where address not
-        interface desc limit 1;
+          like '%:%' and address not like '127%' and address not like '169%' order by
-      - SELECT address AS endpoint_ip2 FROM interface_addresses where address not
+          interface desc limit 1;
-        like '%:%' and address not like '127%' and address not like '169%' order by
+        - SELECT address AS endpoint_ip2 FROM interface_addresses where address not
-        interface asc limit 1;
+          like '%:%' and address not like '127%' and address not like '169%' order by
-      - SELECT hardware_serial FROM system_info;
+          interface asc limit 1;
-      - SELECT hostname AS hostname FROM system_info;
+        - SELECT hardware_serial FROM system_info;
-    options:
+        - SELECT hostname AS hostname FROM system_info;
-      decorations_top_level: true
+      options:
-      disable_distributed: false
+        decorations_top_level: true
-      distributed_interval: 10
+        disable_distributed: false
-      distributed_plugin: tls
+        distributed_interval: 10
-      distributed_tls_max_attempts: 3
+        distributed_plugin: tls
-      distributed_tls_read_endpoint: /api/v1/osquery/distributed/read
+        distributed_tls_max_attempts: 3
-      distributed_tls_write_endpoint: /api/v1/osquery/distributed/write
+        distributed_tls_read_endpoint: /api/v1/osquery/distributed/read
-      enable_windows_events_publisher: true
+        distributed_tls_write_endpoint: /api/v1/osquery/distributed/write
-      enable_windows_events_subscriber: true
+        enable_windows_events_publisher: true
-      logger_plugin: tls
+        enable_windows_events_subscriber: true
-      logger_tls_endpoint: /api/v1/osquery/log
+        logger_plugin: tls
-      logger_tls_period: 10
+        logger_tls_endpoint: /api/v1/osquery/log
-      pack_delimiter: _
+        logger_tls_period: 10
-  overrides: {}
+        pack_delimiter: _
+  server_settings:
+    enable_analytics: false

salt/fleet/init.sls

@@ -17,6 +17,7 @@
 
 
 include:
+  - ssl
   - mysql
 
 # Fleet Setup
@@ -114,20 +115,20 @@ so-fleet:
     - port_bindings:
       - 0.0.0.0:8080:8080
     - environment:
-      - KOLIDE_MYSQL_ADDRESS={{ MAINIP }}:3306
+      - FLEET_MYSQL_ADDRESS={{ MAINIP }}:3306
-      - KOLIDE_REDIS_ADDRESS={{ MAINIP }}:6379
+      - FLEET_REDIS_ADDRESS={{ MAINIP }}:6379
-      - KOLIDE_MYSQL_DATABASE=fleet
+      - FLEET_MYSQL_DATABASE=fleet
-      - KOLIDE_MYSQL_USERNAME=fleetdbuser
+      - FLEET_MYSQL_USERNAME=fleetdbuser
-      - KOLIDE_MYSQL_PASSWORD={{ FLEETPASS }}
+      - FLEET_MYSQL_PASSWORD={{ FLEETPASS }}
-      - KOLIDE_SERVER_CERT=/ssl/server.cert
+      - FLEET_SERVER_CERT=/ssl/server.cert
-      - KOLIDE_SERVER_KEY=/ssl/server.key
+      - FLEET_SERVER_KEY=/ssl/server.key
-      - KOLIDE_LOGGING_JSON=true
+      - FLEET_LOGGING_JSON=true
-      - KOLIDE_AUTH_JWT_KEY= {{ FLEETJWT }}
+      - FLEET_AUTH_JWT_KEY= {{ FLEETJWT }}
-      - KOLIDE_OSQUERY_STATUS_LOG_FILE=/var/log/fleet/status.log
+      - FLEET_FILESYSTEM_STATUS_LOG_FILE=/var/log/fleet/status.log
-      - KOLIDE_OSQUERY_RESULT_LOG_FILE=/var/log/osquery/result.log
+      - FLEET_FILESYSTEM_RESULT_LOG_FILE=/var/log/osquery/result.log
-      - KOLIDE_SERVER_URL_PREFIX=/fleet
+      - FLEET_SERVER_URL_PREFIX=/fleet
-      - KOLIDE_FILESYSTEM_ENABLE_LOG_ROTATION=true
+      - FLEET_FILESYSTEM_ENABLE_LOG_ROTATION=true
-      - KOLIDE_FILESYSTEM_ENABLE_LOG_COMPRESSION=true
+      - FLEET_FILESYSTEM_ENABLE_LOG_COMPRESSION=true
     - binds:
       - /etc/pki/fleet.key:/ssl/server.key:ro
       - /etc/pki/fleet.crt:/ssl/server.cert:ro
@@ -136,10 +137,13 @@ so-fleet:
       - /opt/so/conf/fleet/packs:/packs
     - watch:
       - /opt/so/conf/fleet/etc
+    - require:
+      - x509: fleet_key
+      - x509: fleet_crt
 
 append_so-fleet_so-status.conf:
   file.append:
     - name: /opt/so/conf/so-status/so-status.conf
     - text: so-fleet
 
-{% endif %}
+{% endif %}

salt/grafana/defaults.yaml

@@ -294,7 +294,7 @@ grafana:
             y: 1
             h: 4
             w: 4
-        logstash_estimated_eps_stat:
+        logstash_estimated_eps_in_stat:
           gridPos:
             x: 0
             y: 5
@@ -536,7 +536,7 @@ grafana:
             y: 152
             h: 1
             w: 24
-        logstash_estimated_eps_graph:
+        logstash_estimated_eps_in_graph:
           gridPos:
             x: 0
             y: 153
@@ -598,19 +598,13 @@ grafana:
             x: 0
             y: 188
             h: 8
-            w: 10
+            w: 12
         zeek_capture_loss_graph:
           gridPos:
-            x: 10
+            x: 12
             y: 188
             h: 8
-            w: 10
+            w: 12
-        zeek_restarts_healthcheck_stat:
-          gridPos:
-            x: 20
-            y: 188
-            h: 8
-            w: 4
 
         row_suricata:
           gridPos:
@@ -726,15 +720,9 @@ grafana:
             y: 1
             h: 4
             w: 4
-        logstash_estimated_eps_stat:
-          gridPos:
-            x: 0
-            y: 5
-            h: 4
-            w: 4
         redis_queue_stat:
           gridPos:
-            x: 4
+            x: 0
             y: 5
             h: 4
             w: 4
@@ -920,73 +908,60 @@ grafana:
             h: 8
             w: 12
 
-        row_logstash:
+        row_elasticsearch:
           gridPos:
             x: 0
             y: 152
             h: 1
             w: 24
-        logstash_estimated_eps_graph:
-          gridPos:
-            x: 0
-            y: 153
-            h: 8
-            w: 24
-
-        row_elasticsearch:
-          gridPos:
-            x: 0
-            y: 161
-            h: 1
-            w: 24
         elasticsearch_document_count_graph:
           gridPos:
             x: 0
-            y: 162
+            y: 153
             h: 8
             w: 12
         elasticsearch_thread_count_graph:
           gridPos:
             x: 12
-            y: 162
+            y: 153
             h: 8
             w: 12
         elasticsearch_store_size_graph:
           gridPos:
             x: 0
-            y: 170
+            y: 161
             h: 8
             w: 12
         elasticsearch_field_data_cache_size_graph:
           gridPos:
             x: 12
-            y: 170
+            y: 161
             h: 8
             w: 12
 
         row_redis:
           gridPos:
             x: 0
-            y: 178
+            y: 169
             h: 1
             w: 24
         redis_queue_graph:
           gridPos:
             x: 0
-            y: 179
+            y: 170
             h: 8
             w: 24
 
         row_influxdb:
           gridPos:
             x: 0
-            y: 214
+            y: 178
             h: 1
             w: 24
         influxdb_db_size_graph:
           gridPos:
             x: 0
-            y: 214
+            y: 179
             h: 8
             w: 24
             
@@ -1059,7 +1034,7 @@ grafana:
             y: 1
             h: 4
             w: 4
-        logstash_estimated_eps_stat:
+        logstash_estimated_eps_in_stat:
           gridPos:
             x: 0
             y: 5
@@ -1259,7 +1234,7 @@ grafana:
             y: 152
             h: 1
             w: 24
-        logstash_estimated_eps_graph:
+        logstash_estimated_eps_in_graph:
           gridPos:
             x: 0
             y: 153
@@ -1510,175 +1485,176 @@ grafana:
             y: 61
             h: 8
             w: 24
-        monitor_interface_packets_graph:
+        monitor_interface_traffic_inbound_total_graph:
           gridPos:
             x: 0
             y: 69
             h: 8
+            w: 24
+        monitor_interface_packets_graph:
+          gridPos:
+            x: 0
+            y: 77
+            h: 8
             w: 12
         monitor_interface_drops_graph:
           gridPos:
             x: 12
-            y: 69
+            y: 77
             h: 8
             w: 12
 
         row_disk_usage:
           gridPos:
             x: 0
-            y: 77
+            y: 85
             h: 1
             w: 24
         disk_usage_root_graph:
           gridPos:
             x: 0
-            y: 78
+            y: 86
             h: 8
             w: 12
         disk_usage_nsm_graph:
           gridPos:
             x: 12
-            y: 78
+            y: 86
             h: 8
             w: 12
 
         row_disk_iops:
           gridPos:
             x: 0
-            y: 86
+            y: 94
             h: 1
             w: 24
         disk_io_requests_graph:
           gridPos:
             x: 0
-            y: 87
+            y: 95
             h: 8
             w: 8
         disk_io_bytes_graph:
           gridPos:
             x: 8
-            y: 87
+            y: 95
             h: 8
             w: 8
         disk_io_time_graph:
           gridPos:
             x: 16
-            y: 87
+            y: 95
             h: 8
             w: 8
 
         row_docker_details:
           gridPos:
             x: 0
-            y: 95
+            y: 103
             h: 1
             w: 24
         cpu_docker_combined_current_graph:
           gridPos:
             x: 0
-            y: 96
+            y: 104
             h: 8
             w: 24
         cpu_docker_combined_trend_graph:
           gridPos:
             x: 0
-            y: 104
+            y: 112
             h: 8
             w: 24
         memory_used_docker_combined_current_graph:
           gridPos:
             x: 0
-            y: 112
+            y: 120
             h: 8
             w: 24
         memory_used_docker_combined_trend_graph:
           gridPos:
             x: 0
-            y: 120
+            y: 128
             h: 8
             w: 24
         network_usage_docker_combined_current_graph:
           gridPos:
             x: 0
-            y: 128
+            y: 136
             h: 8
             w: 24
         network_usage_docker_combined_trend_graph:
           gridPos:
             x: 0
-            y: 136
+            y: 144
             h: 8
             w: 24
         uptime_docker_combined_current_graph:
           gridPos:
             x: 0
-            y: 144
+            y: 152
             h: 8
             w: 12
         uptime_docker_combined_trend_graph:
           gridPos:
             x: 12
-            y: 144
+            y: 152
             h: 8
             w: 12
 
         row_zeek:
           gridPos:
             x: 0
-            y: 152
+            y: 160
             h: 1
             w: 24
         zeek_packet_loss_graph:
           gridPos:
             x: 0
-            y: 153
+            y: 161
             h: 8
-            w: 10
+            w: 12
         zeek_capture_loss_graph:
           gridPos:
-            x: 10
+            x: 12
-            y: 153
+            y: 161
             h: 8
-            w: 10
+            w: 12
-        zeek_restarts_healthcheck_stat:
-          gridPos:
-            x: 20
-            y: 153
-            h: 8
-            w: 4
 
         row_suricata:
           gridPos:
             x: 0
-            y: 161
+            y: 169
             h: 1
             w: 24
         suricata_packet_loss_graph:
           gridPos:
             x: 0
-            y: 162
+            y: 170
             h: 8
             w: 24
 
         row_stenographer:
           gridPos:
             x: 0
-            y: 170
+            y: 178
             h: 1
             w: 24
         stenographer_packet_loss_graph:
           gridPos:
             x: 0
-            y: 171
+            y: 179
             h: 8
             w: 16
         stenographer_pcap_retention_graph:
           gridPos:
             x: 16
-            y: 171
+            y: 179
             h: 8
             w: 8
 
+
     searchnode:
       templating:
         list:
@@ -1747,13 +1723,13 @@ grafana:
             y: 1
             h: 4
             w: 4
-        logstash_estimated_eps_stat:
+        logstash_estimated_eps_in_stat:
           gridPos:
             x: 0
             y: 5
             h: 4
             w: 4
-        redis_queue_stat:
+        logstash_estimated_eps_out_stat:
           gridPos:
             x: 4
             y: 5
@@ -1947,23 +1923,28 @@ grafana:
             y: 152
             h: 1
             w: 24
-        logstash_estimated_eps_graph:
+        logstash_estimated_eps_in_graph:
           gridPos:
             x: 0
             y: 153
             h: 8
             w: 24
-
+        logstash_estimated_eps_in_total_graph:
-        row_redis:
           gridPos:
             x: 0
             y: 161
-            h: 1
+            h: 8
             w: 24
-        redis_queue_graph:
+        logstash_estimated_eps_out_graph:
           gridPos:
             x: 0
-            y: 162
+            y: 169
+            h: 8
+            w: 24
+        logstash_estimated_eps_out_total_graph:
+          gridPos:
+            x: 0
+            y: 172
             h: 8
             w: 24
 
@@ -2042,39 +2023,33 @@ grafana:
             y: 1
             h: 4
             w: 4
-        logstash_estimated_eps_stat:
+        logstash_estimated_eps_in_stat:
           gridPos:
             x: 0
             y: 5
             h: 4
             w: 4
-        redis_queue_stat:
+        monitor_interface_traffic_stat:
           gridPos:
             x: 4
             y: 5
             h: 4
             w: 4
-        monitor_interface_traffic_stat:
+        zeek_packet_loss_stat:
           gridPos:
             x: 8
             y: 5
             h: 4
             w: 4
-        zeek_packet_loss_stat:
+        suricata_packet_loss_stat:
           gridPos:
             x: 12
             y: 5
             h: 4
             w: 4
-        suricata_packet_loss_stat:
-          gridPos:
-            x: 16
-            y: 5
-            h: 4
-            w: 4
         stenographer_packet_loss_stat:
           gridPos:
-            x: 20
+            x: 16
             y: 5
             h: 4
             w: 4
@@ -2284,26 +2259,13 @@ grafana:
             y: 152
             h: 1
             w: 24
-        logstash_estimated_eps_graph:
+        logstash_estimated_eps_in_graph:
           gridPos:
             x: 0
             y: 153
             h: 8
             w: 24
 
-        row_redis:
-          gridPos:
-            x: 0
-            y: 161
-            h: 1
-            w: 24
-        redis_queue_graph:
-          gridPos:
-            x: 0
-            y: 162
-            h: 8
-            w: 24
-
         row_zeek:
           gridPos:
             x: 0
@@ -2315,19 +2277,13 @@ grafana:
             x: 0
             y: 171
             h: 8
-            w: 10
+            w: 12
         zeek_capture_loss_graph:
           gridPos:
-            x: 10
+            x: 12
             y: 171
             h: 8
-            w: 10
+            w: 12
-        zeek_restarts_healthcheck_stat:
-          gridPos:
-            x: 20
-            y: 171
-            h: 8
-            w: 4
 
         row_suricata:
           gridPos:
@@ -2721,19 +2677,13 @@ grafana:
             x: 0
             y: 188
             h: 8
-            w: 10
+            w: 12
         zeek_capture_loss_graph:
           gridPos:
-            x: 10
+            x: 12
             y: 188
             h: 8
-            w: 10
+            w: 12
-        zeek_restarts_healthcheck_stat:
-          gridPos:
-            x: 20
-            y: 188
-            h: 8
-            w: 4
 
         row_suricata:
           gridPos:
@@ -2779,3 +2729,107 @@ grafana:
             y: 214
             h: 8
             w: 24
+
+
+    pipeline_overview_nontc:
+      title: 'Pipeline Overview'
+      templating:
+        list:
+          searchnode:
+            includeAll: true
+            multi: true
+            hide: 2
+            text: All
+            value: "$__all"
+      panels:
+        redis_queue_graph:
+          gridPos:
+            x: 0
+            y: 0
+            h: 8
+            w: 8
+        logstash_eps_in_out_manager_graph:
+          gridPos:
+            x: 8
+            y: 0
+            h: 8
+            w: 8
+        logstash_indexing_eps_in_searchnode_total_graph:
+          gridPos:
+            x: 16
+            y: 0
+            h: 8
+            w: 8
+        logstash_indexing_eps_in_out_searchnode_graph:
+          gridPos:
+            x: 0
+            y: 8
+            h: 8
+            w: 24
+        elasticsearch_ingest_performance_nontc_graph:
+          gridPos:
+            x: 0
+            y: 16
+            h: 8
+            w: 24
+        elasticsearch_pipeline_time_nontc_graph:
+          gridPos:
+            x: 0
+            y: 24
+            h: 8
+            w: 24
+
+
+    pipeline_overview_tc:
+      title: 'Pipeline Overview'
+      templating:
+        list:
+          searchnode:
+            includeAll: true
+            multi: true
+            hide: 2
+            text: All
+            value: "$__all"
+          cluster_name:
+            includeAll: true
+            multi: true
+            hide: 2
+            text: All
+            value: "$__all"
+      panels:
+        redis_queue_graph:
+          gridPos:
+            x: 0
+            y: 0
+            h: 8
+            w: 8
+        logstash_eps_in_out_manager_graph:
+          gridPos:
+            x: 8
+            y: 0
+            h: 8
+            w: 8
+        logstash_indexing_eps_in_searchnode_total_graph:
+          gridPos:
+            x: 16
+            y: 0
+            h: 8
+            w: 8
+        logstash_indexing_eps_in_out_searchnode_graph:
+          gridPos:
+            x: 0
+            y: 8
+            h: 8
+            w: 24
+        elasticsearch_ingest_performance_tc_graph:
+          gridPos:
+            x: 0
+            y: 16
+            h: 8
+            w: 24
+        elasticsearch_pipeline_time_tc_graph:
+          gridPos:
+            x: 0
+            y: 24
+            h: 8
+            w: 24

salt/grafana/init.sls

@@ -17,6 +17,11 @@
 {% if grains.role == 'so-eval' %}
   {% do DASHBOARDS.append('eval') %}
 {% else %}
+  {% if not salt['pillar.get']('elasticsearch:true_cluster', False) %}
+    {% do DASHBOARDS.append('pipeline_overview_nontc') %}
+  {% else %}
+    {% do DASHBOARDS.append('pipeline_overview_tc') %}
+  {% endif %}
   {# Grab a unique listing of nodetypes that exists so that we create only the needed dashboards #}
   {% for dashboard in salt['cmd.shell']("ls /opt/so/saltstack/local/pillar/minions/|awk -F'_' {'print $2'}|awk -F'.' {'print $1'}").split() %}
     {% if dashboard in ALLOWED_DASHBOARDS %}
@@ -132,6 +137,8 @@ so-grafana:
       - 0.0.0.0:3000:3000
     - watch:
       - file: /opt/so/conf/grafana/*
+    - require:
+      - file: grafana-config
 
 append_so-grafana_so-status.conf:
   file.append:

salt/grafana/panels/cpu_docker_combined_current_graph.json.jinja

@@ -1,20 +1,151 @@
 {
-  "type": "graph",
+  "id": 100,
-  "title": "Container CPU Usage Current",
   "gridPos": {
     "x": {{ PANELS.cpu_docker_combined_current_graph.gridPos.x }},
     "y": {{ PANELS.cpu_docker_combined_current_graph.gridPos.y }},
     "w": {{ PANELS.cpu_docker_combined_current_graph.gridPos.w }},
     "h": {{ PANELS.cpu_docker_combined_current_graph.gridPos.h }}
   },
-  "id": 100,
+  "type": "timeseries",
+  "title": "Container CPU Usage Current",
+  "transformations": [],
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 10,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "decimals": 1,
+      "unit": "percent"
+    },
+    "overrides": [
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/n_cpus/"
+        },
+        "properties": [
+          {
+            "id": "custom.fillOpacity",
+            "value": 0
+          },
+          {
+            "id": "color",
+            "value": {
+              "mode": "fixed",
+              "fixedColor": "dark-red"
+            }
+          }
+        ]
+      }
+    ]
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
   "targets": [
     {
-      "refId": "A",
+      "alias": "$tag_host: $tag_container_name",
-      "queryType": "randomWalk",
+      "groupBy": [
-      "policy": "default",
+        {
-      "resultFormat": "time_series",
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "container_name"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "host"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "hide": false,
+      "measurement": "docker_container_cpu",
       "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT mean(\"usage_percent\") FROM \"docker_container_cpu\" WHERE (\"host\" =~ /^$servername$/ AND \"container_name\" =~ /^$containers$/) AND $timeFilter GROUP BY time($__interval), \"container_name\", \"host\" fill(null)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "A",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "usage_percent"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
       "tags": [
         {
           "key": "host",
@@ -27,131 +158,66 @@
           "operator": "=~",
           "value": "/^$containers$/"
         }
-      ],
+      ]
+    },
+    {
+      "alias": "$tag_host: n_cpus*100",
       "groupBy": [
         {
-          "type": "time",
           "params": [
             "$__interval"
-          ]
+          ],
+          "type": "time"
         },
         {
-          "type": "tag",
           "params": [
-            "container_name"
+            "host"
-          ]
+          ],
+          "type": "tag"
         },
         {
-          "type": "fill",
           "params": [
             "null"
-          ]
+          ],
+          "type": "fill"
         }
       ],
+      "hide": false,
+      "measurement": "system",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "refId": "B",
+      "resultFormat": "time_series",
       "select": [
         [
           {
-            "type": "field",
             "params": [
-              "usage_percent"
+              "n_cpus"
-            ]
+            ],
+            "type": "field"
           },
           {
-            "type": "mean",
+            "params": [],
-            "params": []
+            "type": "last"
           },
           {
-            "type": "math",
             "params": [
-              " / $cpucount"
+              " * 100"
-            ]
+            ],
+            "type": "math"
           }
         ]
       ],
-      "measurement": "docker_container_cpu",
+      "tags": [
-      "alias": "$tag_container_name"
+        {
+          "key": "host",
+          "operator": "=~",
+          "value": "/^$servername$/"
+        }
+      ]
     }
   ],
-  "options": {
+  "maxDataPoints": null,
-    "alertThreshold": true
-  },
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
-  "pluginVersion": "7.5.4",
-  "renderer": "flot",
-  "yaxes": [
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": null,
-      "max": null,
-      "format": "percent",
-      "$$hashKey": "object:315"
-    },
-    {
-      "label": null,
-      "show": false,
-      "logBase": 1,
-      "min": null,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:316"
-    }
-  ],
-  "xaxis": {
-    "show": true,
-    "mode": "time",
-    "name": null,
-    "values": [],
-    "buckets": null
-  },
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "lines": true,
-  "fill": 1,
-  "linewidth": 1,
-  "dashLength": 10,
-  "spaceLength": 10,
-  "pointradius": 2,
-  "legend": {
-    "show": true,
-    "values": false,
-    "min": false,
-    "max": false,
-    "current": false,
-    "total": false,
-    "avg": false,
-    "alignAsTable": false,
-    "rightSide": false,
-    "hideZero": false
-  },
-  "nullPointMode": "connected",
-  "tooltip": {
-    "value_type": "individual",
-    "shared": true,
-    "sort": 2
-  },
-  "aliasColors": {},
-  "seriesOverrides": [],
-  "thresholds": [],
-  "timeRegions": [],
-  "decimals": null,
-  "fillGradient": 0,
-  "dashes": false,
-  "hiddenSeries": false,
-  "points": false,
-  "bars": false,
-  "stack": false,
-  "percentage": false,
-  "steppedLine": false,
   "timeFrom": null,
-  "timeShift": null,
+  "timeShift": null
-  "maxDataPoints": 750,
-  "interval": "30s"
 }

salt/grafana/panels/cpu_docker_combined_trend_graph.json.jinja

@@ -1,20 +1,147 @@
 {
-  "type": "graph",
+  "id": 101,
-  "title": "Container CPU Usage Trend",
   "gridPos": {
     "x": {{ PANELS.cpu_docker_combined_trend_graph.gridPos.x }},
     "y": {{ PANELS.cpu_docker_combined_trend_graph.gridPos.y }},
     "w": {{ PANELS.cpu_docker_combined_trend_graph.gridPos.w }},
     "h": {{ PANELS.cpu_docker_combined_trend_graph.gridPos.h }}
   },
-  "id": 101,
+  "type": "timeseries",
+  "title": "Container CPU Usage Trend",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 10,
+        "gradientMode": "none",
+        "spanNulls": true,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "decimals": 1,
+      "unit": "percent"
+    },
+    "overrides": [
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/n_cpus/"
+        },
+        "properties": [
+          {
+            "id": "custom.fillOpacity",
+            "value": 0
+          },
+          {
+            "id": "color",
+            "value": {
+              "mode": "fixed",
+              "fixedColor": "dark-red"
+            }
+          }
+        ]
+      }
+    ]
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
   "targets": [
     {
-      "refId": "A",
+      "alias": "$tag_host: $tag_container_name",
-      "queryType": "randomWalk",
+      "groupBy": [
-      "policy": "so_long_term",
+        {
-      "resultFormat": "time_series",
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "container_name"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "host"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "measurement": "docker_container_cpu",
       "orderByTime": "ASC",
+      "policy": "so_long_term",
+      "queryType": "randomWalk",
+      "refId": "A",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "mean_usage_percent"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
       "tags": [
         {
           "key": "host",
@@ -27,132 +154,67 @@
           "operator": "=~",
           "value": "/^$containers$/"
         }
-      ],
+      ]
+    },
+    {
+      "alias": "$tag_host: n_cpus*100",
       "groupBy": [
         {
-          "type": "time",
           "params": [
             "$__interval"
-          ]
+          ],
+          "type": "time"
         },
         {
-          "type": "tag",
           "params": [
-            "container_name"
+            "host"
-          ]
+          ],
+          "type": "tag"
         },
         {
-          "type": "fill",
           "params": [
             "null"
-          ]
+          ],
+          "type": "fill"
         }
       ],
+      "hide": false,
+      "measurement": "system",
+      "orderByTime": "ASC",
+      "policy": "so_long_term",
+      "refId": "B",
+      "resultFormat": "time_series",
       "select": [
         [
           {
-            "type": "field",
             "params": [
-              "mean_usage_percent"
+              "mean_n_cpus"
-            ]
+            ],
+            "type": "field"
           },
           {
-            "type": "mean",
+            "params": [],
-            "params": []
+            "type": "last"
           },
           {
-            "type": "math",
             "params": [
-              " / $cpucount"
+              " * 100"
-            ]
+            ],
+            "type": "math"
           }
         ]
       ],
-      "measurement": "docker_container_cpu",
+      "tags": [
-      "alias": "$tag_container_name"
+        {
+          "key": "host",
+          "operator": "=~",
+          "value": "/^$servername$/"
+        }
+      ]
     }
   ],
-  "options": {
-    "alertThreshold": true
-  },
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
-  "pluginVersion": "7.5.4",
-  "renderer": "flot",
-  "yaxes": [
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": null,
-      "max": null,
-      "format": "percent",
-      "$$hashKey": "object:315"
-    },
-    {
-      "label": null,
-      "show": false,
-      "logBase": 1,
-      "min": null,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:316"
-    }
-  ],
-  "xaxis": {
-    "show": true,
-    "mode": "time",
-    "name": null,
-    "values": [],
-    "buckets": null
-  },
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "lines": true,
-  "fill": 1,
-  "linewidth": 1,
-  "dashLength": 10,
-  "spaceLength": 10,
-  "pointradius": 2,
-  "legend": {
-    "show": true,
-    "values": true,
-    "min": false,
-    "max": false,
-    "current": false,
-    "total": false,
-    "avg": true,
-    "alignAsTable": false,
-    "rightSide": false,
-    "hideZero": false
-  },
-  "nullPointMode": "connected",
-  "tooltip": {
-    "value_type": "individual",
-    "shared": true,
-    "sort": 2
-  },
-  "aliasColors": {},
-  "seriesOverrides": [],
-  "thresholds": [],
-  "timeRegions": [],
-  "decimals": 1,
-  "fillGradient": 0,
-  "dashes": false,
-  "hiddenSeries": false,
-  "points": false,
-  "bars": false,
-  "stack": false,
-  "percentage": false,
-  "steppedLine": false,
-  "timeFrom": null,
-  "timeShift": null,
   "description": "",
-  "maxDataPoints": 750,
+  "maxDataPoints": null,
-  "interval": "30s"
+  "timeFrom": null,
+  "timeShift": null
 }

salt/grafana/panels/cpu_usage_current_graph.json.jinja

@@ -1,47 +1,79 @@
 {
-  "aliasColors": {},
+  "id": 69001,
-  "dashLength": 10,
-  "datasource": "InfluxDB",
-  "decimals": 1,
-  "fieldConfig": {
-    "defaults": {
-      "unit": "percent"
-    },
-    "overrides": []
-  },
   "gridPos": {
     "x": {{ PANELS.cpu_usage_current_graph.gridPos.x }},
     "y": {{ PANELS.cpu_usage_current_graph.gridPos.y }},
     "w": {{ PANELS.cpu_usage_current_graph.gridPos.w }},
     "h": {{ PANELS.cpu_usage_current_graph.gridPos.h }}
   },
-  "id": 69001,
+  "type": "timeseries",
+  "title": "CPU Usage",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
   "interval": "30",
-  "legend": {
+  "fieldConfig": {
-    "alignAsTable": true,
+    "defaults": {
-    "avg": true,
+      "custom": {
-    "current": true,
+        "drawStyle": "line",
-    "max": true,
+        "lineInterpolation": "linear",
-    "min": false,
+        "barAlignment": 0,
-    "rightSide": true,
+        "lineWidth": 1,
-    "show": true,
+        "fillOpacity": 0,
-    "sort": "current",
+        "gradientMode": "none",
-    "sortDesc": true,
+        "spanNulls": false,
-    "total": false,
+        "showPoints": "never",
-    "values": true
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "percent",
+      "min": 0,
+      "decimals": 1
+    },
+    "overrides": []
   },
-  "lines": true,
-  "linewidth": 1,
-  "maxDataPoints": 750,
-  "nullPointMode": "connected",
   "options": {
-    "alertThreshold": true
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
   },
-  "pluginVersion": "7.5.4",
-  "pointradius": 2,
-  "renderer": "flot",
-  "seriesOverrides": [],
-  "spaceLength": 10,
   "targets": [
     {
       "alias": "$tag_host $tag_role",
@@ -59,10 +91,10 @@
           "type": "tag"
         },
         {
-          "type": "tag",
           "params": [
             "role"
-          ]
+          ],
+          "type": "tag"
         },
         {
           "params": [
@@ -80,20 +112,20 @@
       "select": [
         [
           {
-            "type": "field",
             "params": [
               "usage_idle"
-            ]
+            ],
+            "type": "field"
           },
           {
-            "type": "mean",
+            "params": [],
-            "params": []
+            "type": "mean"
           },
           {
-            "type": "math",
             "params": [
               "*-1 + 100"
-            ]
+            ],
+            "type": "math"
           }
         ]
       ],
@@ -112,55 +144,7 @@
       ]
     }
   ],
-  "thresholds": [],
+  "maxDataPoints": null,
-  "timeRegions": [],
-  "title": "CPU Usage",
-  "tooltip": {
-    "shared": true,
-    "sort": 2,
-    "value_type": "individual"
-  },
-  "type": "graph",
-  "xaxis": {
-    "buckets": null,
-    "mode": "time",
-    "name": null,
-    "show": true,
-    "values": []
-  },
-  "yaxes": [
-    {
-      "$$hashKey": "object:933",
-      "format": "percent",
-      "label": null,
-      "logBase": 1,
-      "max": null,
-      "min": "0",
-      "show": true
-    },
-    {
-      "$$hashKey": "object:934",
-      "format": "short",
-      "label": null,
-      "logBase": 1,
-      "max": null,
-      "min": null,
-      "show": true
-    }
-  ],
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "bars": false,
-  "dashes": false,
-  "fill": 0,
-  "fillGradient": 0,
-  "hiddenSeries": false,
-  "percentage": false,
-  "points": false,
-  "stack": false,
-  "steppedLine": false,
   "timeFrom": null,
   "timeShift": null
 }

salt/grafana/panels/cpu_usage_guage.json.jinja

@@ -1,65 +1,69 @@
 {
-  "cacheTimeout": null,
-  "colorBackground": false,
-  "colorValue": true,
-  "colors": [
-    "rgba(50, 172, 45, 0.97)",
-    "rgba(237, 129, 40, 0.89)",
-    "rgba(245, 54, 54, 0.9)"
-  ],
-  "datasource": "InfluxDB",
-  "editable": true,
-  "error": false,
-  "format": "percent",
-  "gauge": {
-    "maxValue": 100,
-    "minValue": 0,
-    "show": true,
-    "thresholdLabels": false,
-    "thresholdMarkers": true
-  },
-    "gridPos": {
-        "x": {{ PANELS.cpu_usage_guage.gridPos.x }},
-        "y": {{ PANELS.cpu_usage_guage.gridPos.y }},
-        "w": {{ PANELS.cpu_usage_guage.gridPos.w }},
-        "h": {{ PANELS.cpu_usage_guage.gridPos.h }}
-      },
-  "height": "150",
   "id": 9,
-  "interval": null,
+  "gridPos": {
-  "links": [],
+    "x": {{ PANELS.cpu_usage_guage.gridPos.x }},
-  "mappingType": 1,
+    "y": {{ PANELS.cpu_usage_guage.gridPos.y }},
-  "mappingTypes": [
+    "w": {{ PANELS.cpu_usage_guage.gridPos.w }},
-    {
+    "h": {{ PANELS.cpu_usage_guage.gridPos.h }}
-      "name": "value to text",
+  },
-      "value": 1
+  "type": "gauge",
-    },
+  "title": "CPU usage",
-    {
+  "datasource": "InfluxDB",
-      "name": "range to text",
+  "pluginVersion": "8.2.1",
-      "value": 2
+  "links": [],
-    }
+  "fieldConfig": {
-  ],
+    "defaults": {
-  "maxDataPoints": 100,
+      "thresholds": {
-  "nullPointMode": "connected",
+        "mode": "absolute",
-  "nullText": null,
+        "steps": [
-  "postfix": "",
+          {
-  "postfixFontSize": "50%",
+            "color": "rgba(50, 172, 45, 0.97)",
-  "prefix": "",
+            "value": null
-  "prefixFontSize": "50%",
+          },
-  "rangeMaps": [
+          {
-    {
+            "color": "rgba(237, 129, 40, 0.89)",
-      "from": "null",
+            "value": 70
-      "text": "N/A",
+          },
-      "to": "null"
+          {
-    }
+            "color": "rgba(245, 54, 54, 0.9)",
-  ],
+            "value": 80
-  "sparkline": {
+          }
-    "fillColor": "rgba(31, 118, 189, 0.18)",
+        ]
-    "full": false,
+      },
-    "lineColor": "rgb(31, 120, 193)",
+      "mappings": [
-    "show": false
+        {
+          "options": {
+            "match": "null",
+            "result": {
+              "text": "N/A"
+            }
+          },
+          "type": "special"
+        }
+      ],
+      "color": {
+        "mode": "thresholds"
+      },
+      "max": 100,
+      "min": 0,
+      "unit": "percent"
+    },
+    "overrides": []
+  },
+  "interval": "30",
+  "options": {
+    "reduceOptions": {
+      "values": false,
+      "calcs": [
+        "lastNotNull"
+      ],
+      "fields": ""
+    },
+    "orientation": "horizontal",
+    "showThresholdLabels": false,
+    "showThresholdMarkers": true,
+    "text": {}
   },
-  "tableColumn": "",
   "targets": [
     {
       "dsType": "influxdb",
@@ -104,8 +108,8 @@
       "tags": [
         {
           "key": "host",
-          "operator": "=~",
+          "operator": "=",
-          "value": "/^$servername$/"
+          "value": "$servername"
         },
         {
           "condition": "AND",
@@ -113,23 +117,10 @@
           "operator": "=",
           "value": "cpu-total"
         }
-      ]
+      ],
+      "orderByTime": "ASC"
     }
   ],
-  "thresholds": "70,80,90",
+  "maxDataPoints": null,
-  "title": "CPU usage",
+  "cacheTimeout": null
-  "type": "singlestat",
-  "valueFontSize": "80%",
-  "valueMaps": [
-    {
-      "op": "=",
-      "text": "N/A",
-      "value": "null"
-    }
-  ],
-  "valueName": "current",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  }
 }

salt/grafana/panels/cpu_usage_tasks_all_graph.json.jinja

@@ -1,51 +1,30 @@
 {
-  "aliasColors": {},
+  "id": 61871,
-  "dashLength": 10,
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
-  "fill": 1,
   "gridPos": {
     "x": {{ PANELS.cpu_usage_tasks_all_graph.gridPos.x }},
     "y": {{ PANELS.cpu_usage_tasks_all_graph.gridPos.y }},
     "w": {{ PANELS.cpu_usage_tasks_all_graph.gridPos.w }},
     "h": {{ PANELS.cpu_usage_tasks_all_graph.gridPos.h }}
   },
-  "id": 61871,
+  "type": "timeseries",
-  "legend": {
+  "title": "CPU Usage",
-    "alignAsTable": true,
+  "datasource": "InfluxDB",
-    "avg": true,
+  "pluginVersion": "8.2.1",
-    "current": true,
+  "interval": "30s",
-    "hideEmpty": true,
-    "hideZero": true,
-    "max": true,
-    "min": true,
-    "show": true,
-    "total": false,
-    "values": true
-  },
-  "lines": true,
-  "linewidth": 1,
-  "nullPointMode": "connected",
   "options": {
-    "alertThreshold": true
+    "tooltip": {
-  },
+      "mode": "single"
-  "pluginVersion": "7.5.4",
+    },
-  "pointradius": 2,
+    "legend": {
-  "renderer": "flot",
+      "displayMode": "table",
-  "seriesOverrides": [
+      "placement": "right",
-    {
+      "calcs": [
-      "$$hashKey": "object:266",
+        "max",
-      "alias": "/trend/",
+        "mean",
-      "fill": 0,
+        "lastNotNull"
-      "linewidth": 4,
+      ]
-      "dashes": true,
-      "dashLength": 4
     }
-  ],
+  },
-  "spaceLength": 10,
   "targets": [
     {
       "alias": "$tag_host: $col",
@@ -84,7 +63,8 @@
           }
         ]
       ],
-      "tags": []
+      "tags": [],
+      "hide": false
     },
     {
       "alias": "$tag_host: $col",
@@ -102,9 +82,10 @@
           "type": "fill"
         }
       ],
+      "hide": false,
       "orderByTime": "ASC",
       "policy": "default",
-      "query": "SELECT mean(mean_usage_user) as \"trend_user\", mean(mean_usage_system) as \"trend_system\", mean(mean_usage_softirq) as \"trend_softirq\", mean(mean_usage_steal) as \"trend_steal\", mean(mean_usage_nice) as \"trend_nice\", mean(mean_usage_irq) as \"trend_irq\", mean(mean_usage_iowait) as \"trend_iowait\", mean(mean_usage_guest) as \"trend_guest\", mean(mean_usage_guest_nice) as \"trend_guest_nice\"  FROM \"so_long_term\".\"cpu\" WHERE \"host\" =~ /^$servername$/ and cpu = 'cpu-total' AND $timeFilter GROUP BY time($__interval), *",
+      "query": "SELECT mean(mean_usage_user) as \"trend_user\", mean(mean_usage_system) as \"trend_system\", mean(mean_usage_softirq) as \"trend_softirq\", mean(mean_usage_steal) as \"trend_steal\", mean(mean_usage_nice) as \"trend_nice\", mean(mean_usage_irq) as \"trend_irq\", mean(mean_usage_iowait) as \"trend_iowait\", mean(mean_usage_guest) as \"trend_guest\", mean(mean_usage_guest_nice) as \"trend_guest_nice\"  FROM \"so_long_term\".\"cpu\" WHERE \"host\" =~ /^$servername$/ and cpu = 'cpu-total' AND $timeFilter GROUP BY time($__interval), * fill(linear)",
       "queryType": "randomWalk",
       "rawQuery": true,
       "refId": "B",
@@ -123,61 +104,90 @@
           }
         ]
       ],
-      "tags": [],
+      "tags": []
-      "hide": false
     }
   ],
-  "thresholds": [],
+  "fieldConfig": {
-  "timeRegions": [],
+    "defaults": {
-  "title": "CPU Usage",
+      "custom": {
-  "tooltip": {
+        "drawStyle": "line",
-    "shared": true,
+        "lineInterpolation": "linear",
-    "sort": 2,
+        "barAlignment": 0,
-    "value_type": "individual"
+        "lineWidth": 1,
-  },
+        "fillOpacity": 10,
-  "type": "graph",
+        "gradientMode": "none",
-  "xaxis": {
+        "spanNulls": false,
-    "buckets": null,
+        "showPoints": "never",
-    "mode": "time",
+        "pointSize": 5,
-    "name": null,
+        "stacking": {
-    "show": true,
+          "mode": "none",
-    "values": []
+          "group": "A"
-  },
+        },
-  "yaxes": [
+        "axisPlacement": "auto",
-    {
+        "axisLabel": "",
-      "$$hashKey": "object:202",
+        "scaleDistribution": {
-      "decimals": null,
+          "type": "linear"
-      "format": "percent",
+        },
-      "label": null,
+        "hideFrom": {
-      "logBase": 1,
+          "tooltip": false,
-      "max": "100",
+          "viz": false,
-      "min": "0",
+          "legend": false
-      "show": true
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "percent",
+      "min": 0,
+      "decimals": 1
     },
-    {
+    "overrides": [
-      "$$hashKey": "object:203",
+      {
-      "format": "short",
+        "matcher": {
-      "label": null,
+          "id": "byRegexp",
-      "logBase": 1,
+          "options": "/trend/"
-      "max": null,
+        },
-      "min": null,
+        "properties": [
-      "show": true
+          {
-    }
+            "id": "custom.fillOpacity",
-  ],
+            "value": 0
-  "yaxis": {
+          },
-    "align": false,
+          {
-    "alignLevel": null
+            "id": "custom.lineWidth",
+            "value": 4
+          },
+          {
+            "id": "custom.lineStyle",
+            "value": {
+              "fill": "dash",
+              "dash": [
+                4,
+                10
+              ]
+            }
+          }
+        ]
+      }
+    ]
   },
-  "bars": false,
+  "maxDataPoints": null,
-  "dashes": false,
-  "fillGradient": 0,
-  "hiddenSeries": false,
-  "percentage": false,
-  "points": false,
-  "stack": false,
-  "steppedLine": false,
   "timeFrom": null,
-  "timeShift": null,
+  "timeShift": null
-  "maxDataPoints": 750,
-  "interval": "30s"
 }

salt/grafana/panels/cpu_usage_tasks_blocked_graph.json.jinja

@@ -1,132 +1,124 @@
 {
-  "type": "graph",
+  "id": 69005,
-  "title": "CPU Tasks Blocked",
   "gridPos": {
     "x": {{ PANELS.cpu_usage_tasks_blocked_graph.gridPos.x }},
     "y": {{ PANELS.cpu_usage_tasks_blocked_graph.gridPos.y }},
     "w": {{ PANELS.cpu_usage_tasks_blocked_graph.gridPos.w }},
     "h": {{ PANELS.cpu_usage_tasks_blocked_graph.gridPos.h }}
   },
-  "id": 69005,
+  "type": "timeseries",
+  "title": "CPU Tasks Blocked",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 0,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "short",
+      "min": 0,
+      "decimals": 1
+    },
+    "overrides": []
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
   "targets": [
     {
-      "refId": "A",
+      "alias": "$tag_host $tag_role",
-      "queryType": "randomWalk",
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
       "groupBy": [
         {
-          "type": "time",
           "params": [
             "$__interval"
-          ]
+          ],
+          "type": "time"
         },
         {
-          "type": "fill",
           "params": [
             "null"
-          ]
+          ],
+          "type": "fill"
         }
       ],
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT mean(blocked) as blocked FROM \"processes\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), host, role ORDER BY asc",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "A",
+      "resultFormat": "time_series",
       "select": [
         [
           {
-            "type": "field",
             "params": [
               "value"
-            ]
+            ],
+            "type": "field"
           },
           {
-            "type": "mean",
+            "params": [],
-            "params": []
+            "type": "mean"
           }
         ]
       ],
-      "query": "SELECT mean(blocked) as blocked FROM \"processes\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), host, role ORDER BY asc",
+      "tags": []
-      "rawQuery": true,
-      "alias": "$tag_host $tag_role"
     }
   ],
-  "options": {
-    "alertThreshold": true
-  },
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
-  "pluginVersion": "7.5.4",
-  "renderer": "flot",
-  "yaxes": [
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": 0,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:412"
-    },
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": null,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:413"
-    }
-  ],
-  "xaxis": {
-    "show": true,
-    "mode": "time",
-    "name": null,
-    "values": [],
-    "buckets": null
-  },
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "lines": true,
-  "linewidth": 1,
-  "dashLength": 10,
-  "spaceLength": 10,
-  "pointradius": 2,
-  "legend": {
-    "alignAsTable": true,
-    "avg": true,
-    "current": true,
-    "max": true,
-    "min": false,
-    "rightSide": true,
-    "show": true,
-    "sort": "current",
-    "sortDesc": true,
-    "total": false,
-    "values": true
-  },
-  "nullPointMode": "connected",
-  "tooltip": {
-    "value_type": "individual",
-    "shared": true,
-    "sort": 2
-  },
-  "aliasColors": {},
-  "seriesOverrides": [],
-  "thresholds": [],
-  "timeRegions": [],
-  "fill": 0,
-  "fillGradient": 0,
-  "dashes": false,
-  "hiddenSeries": false,
-  "points": false,
-  "bars": false,
-  "stack": false,
-  "percentage": false,
-  "steppedLine": false,
   "timeFrom": null,
-  "timeShift": null
+  "timeShift": null,
+  "interval": "30s"
 }

salt/grafana/panels/cpu_usage_tasks_paging_graph.json.jinja

@@ -1,132 +1,124 @@
 {
-  "type": "graph",
+  "id": 69008,
-  "title": "CPU Tasks Paging",
   "gridPos": {
     "x": {{ PANELS.cpu_usage_tasks_paging_graph.gridPos.x }},
     "y": {{ PANELS.cpu_usage_tasks_paging_graph.gridPos.y }},
     "w": {{ PANELS.cpu_usage_tasks_paging_graph.gridPos.w }},
     "h": {{ PANELS.cpu_usage_tasks_paging_graph.gridPos.h }}
   },
-  "id": 69008,
+  "type": "timeseries",
+  "title": "CPU Tasks Paging",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 0,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "short",
+      "min": 0,
+      "decimals": 1
+    },
+    "overrides": []
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
   "targets": [
     {
-      "refId": "A",
+      "alias": "$tag_host $tag_role",
-      "queryType": "randomWalk",
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
       "groupBy": [
         {
-          "type": "time",
           "params": [
             "$__interval"
-          ]
+          ],
+          "type": "time"
         },
         {
-          "type": "fill",
           "params": [
             "null"
-          ]
+          ],
+          "type": "fill"
         }
       ],
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT mean(paging) as paging FROM \"processes\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), host, role ORDER BY asc",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "A",
+      "resultFormat": "time_series",
       "select": [
         [
           {
-            "type": "field",
             "params": [
               "value"
-            ]
+            ],
+            "type": "field"
           },
           {
-            "type": "mean",
+            "params": [],
-            "params": []
+            "type": "mean"
           }
         ]
       ],
-      "query": "SELECT mean(paging) as paging FROM \"processes\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), host, role ORDER BY asc",
+      "tags": []
-      "rawQuery": true,
-      "alias": "$tag_host $tag_role"
     }
   ],
-  "options": {
+  "interval": "30s",
-    "alertThreshold": true
-  },
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
-  "pluginVersion": "7.5.4",
-  "renderer": "flot",
-  "yaxes": [
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": 0,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:412"
-    },
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": null,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:413"
-    }
-  ],
-  "xaxis": {
-    "show": true,
-    "mode": "time",
-    "name": null,
-    "values": [],
-    "buckets": null
-  },
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "lines": true,
-  "linewidth": 1,
-  "dashLength": 10,
-  "spaceLength": 10,
-  "pointradius": 2,
-  "legend": {
-    "alignAsTable": true,
-    "avg": true,
-    "current": true,
-    "max": true,
-    "min": false,
-    "rightSide": true,
-    "show": true,
-    "sort": "current",
-    "sortDesc": true,
-    "total": false,
-    "values": true
-  },
-  "nullPointMode": "connected",
-  "tooltip": {
-    "value_type": "individual",
-    "shared": true,
-    "sort": 2
-  },
-  "aliasColors": {},
-  "seriesOverrides": [],
-  "thresholds": [],
-  "timeRegions": [],
-  "fill": 0,
-  "fillGradient": 0,
-  "dashes": false,
-  "hiddenSeries": false,
-  "points": false,
-  "bars": false,
-  "stack": false,
-  "percentage": false,
-  "steppedLine": false,
   "timeFrom": null,
   "timeShift": null
 }

salt/grafana/panels/cpu_usage_tasks_running_graph.json.jinja

@@ -1,132 +1,124 @@
 {
-  "type": "graph",
+  "id": 69003,
-  "title": "CPU Tasks Running",
   "gridPos": {
     "x": {{ PANELS.cpu_usage_tasks_running_graph.gridPos.x }},
     "y": {{ PANELS.cpu_usage_tasks_running_graph.gridPos.y }},
     "w": {{ PANELS.cpu_usage_tasks_running_graph.gridPos.w }},
     "h": {{ PANELS.cpu_usage_tasks_running_graph.gridPos.h }}
   },
-  "id": 69003,
+  "type": "timeseries",
+  "title": "CPU Tasks Running",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 0,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "short",
+      "min": 0,
+      "decimals": 1
+    },
+    "overrides": []
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
   "targets": [
     {
-      "refId": "A",
+      "alias": "$tag_host $tag_role",
-      "queryType": "randomWalk",
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
       "groupBy": [
         {
-          "type": "time",
           "params": [
             "$__interval"
-          ]
+          ],
+          "type": "time"
         },
         {
-          "type": "fill",
           "params": [
             "null"
-          ]
+          ],
+          "type": "fill"
         }
       ],
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT mean(running) as running FROM \"processes\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), host, role ORDER BY asc",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "A",
+      "resultFormat": "time_series",
       "select": [
         [
           {
-            "type": "field",
             "params": [
               "value"
-            ]
+            ],
+            "type": "field"
           },
           {
-            "type": "mean",
+            "params": [],
-            "params": []
+            "type": "mean"
           }
         ]
       ],
-      "query": "SELECT mean(running) as running FROM \"processes\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), host, role ORDER BY asc",
+      "tags": []
-      "rawQuery": true,
-      "alias": "$tag_host $tag_role"
     }
   ],
-  "options": {
-    "alertThreshold": true
-  },
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
-  "pluginVersion": "7.5.4",
-  "renderer": "flot",
-  "yaxes": [
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": 0,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:412"
-    },
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": null,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:413"
-    }
-  ],
-  "xaxis": {
-    "show": true,
-    "mode": "time",
-    "name": null,
-    "values": [],
-    "buckets": null
-  },
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "lines": true,
-  "linewidth": 1,
-  "dashLength": 10,
-  "spaceLength": 10,
-  "pointradius": 2,
-  "legend": {
-    "alignAsTable": true,
-    "avg": true,
-    "current": true,
-    "max": true,
-    "min": false,
-    "rightSide": true,
-    "show": true,
-    "sort": "current",
-    "sortDesc": true,
-    "total": false,
-    "values": true
-  },
-  "nullPointMode": "connected",
-  "tooltip": {
-    "value_type": "individual",
-    "shared": true,
-    "sort": 2
-  },
-  "aliasColors": {},
-  "seriesOverrides": [],
-  "thresholds": [],
-  "timeRegions": [],
-  "fill": 0,
-  "fillGradient": 0,
-  "dashes": false,
-  "hiddenSeries": false,
-  "points": false,
-  "bars": false,
-  "stack": false,
-  "percentage": false,
-  "steppedLine": false,
   "timeFrom": null,
-  "timeShift": null
+  "timeShift": null,
+  "interval": "30s"
 }

salt/grafana/panels/cpu_usage_tasks_sleeping_graph.json.jinja

@@ -1,132 +1,124 @@
 {
-  "type": "graph",
+  "id": 69006,
-  "title": "CPU Tasks Sleeping",
   "gridPos": {
     "x": {{ PANELS.cpu_usage_tasks_sleeping_graph.gridPos.x }},
     "y": {{ PANELS.cpu_usage_tasks_sleeping_graph.gridPos.y }},
     "w": {{ PANELS.cpu_usage_tasks_sleeping_graph.gridPos.w }},
     "h": {{ PANELS.cpu_usage_tasks_sleeping_graph.gridPos.h }}
   },
-  "id": 69006,
+  "type": "timeseries",
+  "title": "CPU Tasks Sleeping",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 0,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "short",
+      "min": 0,
+      "decimals": 1
+    },
+    "overrides": []
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
   "targets": [
     {
-      "refId": "A",
+      "alias": "$tag_host $tag_role",
-      "queryType": "randomWalk",
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
       "groupBy": [
         {
-          "type": "time",
           "params": [
             "$__interval"
-          ]
+          ],
+          "type": "time"
         },
         {
-          "type": "fill",
           "params": [
             "null"
-          ]
+          ],
+          "type": "fill"
         }
       ],
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT mean(sleeping) as sleeping FROM \"processes\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), host, role ORDER BY asc",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "A",
+      "resultFormat": "time_series",
       "select": [
         [
           {
-            "type": "field",
             "params": [
               "value"
-            ]
+            ],
+            "type": "field"
           },
           {
-            "type": "mean",
+            "params": [],
-            "params": []
+            "type": "mean"
           }
         ]
       ],
-      "query": "SELECT mean(sleeping) as sleeping FROM \"processes\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), host, role ORDER BY asc",
+      "tags": []
-      "rawQuery": true,
-      "alias": "$tag_host $tag_role"
     }
   ],
-  "options": {
-    "alertThreshold": true
-  },
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
-  "pluginVersion": "7.5.4",
-  "renderer": "flot",
-  "yaxes": [
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": 0,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:412"
-    },
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": null,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:413"
-    }
-  ],
-  "xaxis": {
-    "show": true,
-    "mode": "time",
-    "name": null,
-    "values": [],
-    "buckets": null
-  },
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "lines": true,
-  "linewidth": 1,
-  "dashLength": 10,
-  "spaceLength": 10,
-  "pointradius": 2,
-  "legend": {
-    "alignAsTable": true,
-    "avg": true,
-    "current": true,
-    "max": true,
-    "min": false,
-    "rightSide": true,
-    "show": true,
-    "sort": "current",
-    "sortDesc": true,
-    "total": false,
-    "values": true
-  },
-  "nullPointMode": "connected",
-  "tooltip": {
-    "value_type": "individual",
-    "shared": true,
-    "sort": 2
-  },
-  "aliasColors": {},
-  "seriesOverrides": [],
-  "thresholds": [],
-  "timeRegions": [],
-  "fill": 0,
-  "fillGradient": 0,
-  "dashes": false,
-  "hiddenSeries": false,
-  "points": false,
-  "bars": false,
-  "stack": false,
-  "percentage": false,
-  "steppedLine": false,
   "timeFrom": null,
-  "timeShift": null
+  "timeShift": null,
+  "interval": "30s"
 }

salt/grafana/panels/cpu_usage_tasks_stopped_graph.json.jinja

@@ -1,132 +1,124 @@
 {
-  "type": "graph",
+  "id": 69007,
-  "title": "CPU Tasks Stopped",
   "gridPos": {
     "x": {{ PANELS.cpu_usage_tasks_stopped_graph.gridPos.x }},
     "y": {{ PANELS.cpu_usage_tasks_stopped_graph.gridPos.y }},
     "w": {{ PANELS.cpu_usage_tasks_stopped_graph.gridPos.w }},
     "h": {{ PANELS.cpu_usage_tasks_stopped_graph.gridPos.h }}
   },
-  "id": 69007,
+  "type": "timeseries",
+  "title": "CPU Tasks Stopped",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 0,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "short",
+      "min": 0,
+      "decimals": 1
+    },
+    "overrides": []
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
   "targets": [
     {
-      "refId": "A",
+      "alias": "$tag_host $tag_role",
-      "queryType": "randomWalk",
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
       "groupBy": [
         {
-          "type": "time",
           "params": [
             "$__interval"
-          ]
+          ],
+          "type": "time"
         },
         {
-          "type": "fill",
           "params": [
             "null"
-          ]
+          ],
+          "type": "fill"
         }
       ],
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT mean(stopped) as stopped FROM \"processes\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), host, role ORDER BY asc",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "A",
+      "resultFormat": "time_series",
       "select": [
         [
           {
-            "type": "field",
             "params": [
               "value"
-            ]
+            ],
+            "type": "field"
           },
           {
-            "type": "mean",
+            "params": [],
-            "params": []
+            "type": "mean"
           }
         ]
       ],
-      "query": "SELECT mean(stopped) as stopped FROM \"processes\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), host, role ORDER BY asc",
+      "tags": []
-      "rawQuery": true,
-      "alias": "$tag_host $tag_role"
     }
   ],
-  "options": {
-    "alertThreshold": true
-  },
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
-  "pluginVersion": "7.5.4",
-  "renderer": "flot",
-  "yaxes": [
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": 0,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:412"
-    },
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": null,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:413"
-    }
-  ],
-  "xaxis": {
-    "show": true,
-    "mode": "time",
-    "name": null,
-    "values": [],
-    "buckets": null
-  },
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "lines": true,
-  "linewidth": 1,
-  "dashLength": 10,
-  "spaceLength": 10,
-  "pointradius": 2,
-  "legend": {
-    "alignAsTable": true,
-    "avg": true,
-    "current": true,
-    "max": true,
-    "min": false,
-    "rightSide": true,
-    "show": true,
-    "sort": "current",
-    "sortDesc": true,
-    "total": false,
-    "values": true
-  },
-  "nullPointMode": "connected",
-  "tooltip": {
-    "value_type": "individual",
-    "shared": true,
-    "sort": 2
-  },
-  "aliasColors": {},
-  "seriesOverrides": [],
-  "thresholds": [],
-  "timeRegions": [],
-  "fill": 0,
-  "fillGradient": 0,
-  "dashes": false,
-  "hiddenSeries": false,
-  "points": false,
-  "bars": false,
-  "stack": false,
-  "percentage": false,
-  "steppedLine": false,
   "timeFrom": null,
-  "timeShift": null
+  "timeShift": null,
+  "interval": "30s"
 }

salt/grafana/panels/cpu_usage_tasks_unknown_graph.json.jinja

@@ -1,132 +1,124 @@
 {
-  "type": "graph",
+  "id": 69009,
-  "title": "CPU Tasks Unknown",
   "gridPos": {
     "x": {{ PANELS.cpu_usage_tasks_unknown_graph.gridPos.x }},
     "y": {{ PANELS.cpu_usage_tasks_unknown_graph.gridPos.y }},
     "w": {{ PANELS.cpu_usage_tasks_unknown_graph.gridPos.w }},
     "h": {{ PANELS.cpu_usage_tasks_unknown_graph.gridPos.h }}
   },
-  "id": 69009,
+  "type": "timeseries",
+  "title": "CPU Tasks Unknown",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 0,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "short",
+      "min": 0,
+      "decimals": 1
+    },
+    "overrides": []
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
   "targets": [
     {
-      "refId": "A",
+      "alias": "$tag_host $tag_role",
-      "queryType": "randomWalk",
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
       "groupBy": [
         {
-          "type": "time",
           "params": [
             "$__interval"
-          ]
+          ],
+          "type": "time"
         },
         {
-          "type": "fill",
           "params": [
             "null"
-          ]
+          ],
+          "type": "fill"
         }
       ],
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT mean(unknown) as unknown FROM \"processes\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), host, role ORDER BY asc",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "A",
+      "resultFormat": "time_series",
       "select": [
         [
           {
-            "type": "field",
             "params": [
               "value"
-            ]
+            ],
+            "type": "field"
           },
           {
-            "type": "mean",
+            "params": [],
-            "params": []
+            "type": "mean"
           }
         ]
       ],
-      "query": "SELECT mean(unknown) as unknown FROM \"processes\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), host, role ORDER BY asc",
+      "tags": []
-      "rawQuery": true,
-      "alias": "$tag_host $tag_role"
     }
   ],
-  "options": {
-    "alertThreshold": true
-  },
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
-  "pluginVersion": "7.5.4",
-  "renderer": "flot",
-  "yaxes": [
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": 0,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:412"
-    },
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": null,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:413"
-    }
-  ],
-  "xaxis": {
-    "show": true,
-    "mode": "time",
-    "name": null,
-    "values": [],
-    "buckets": null
-  },
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "lines": true,
-  "linewidth": 1,
-  "dashLength": 10,
-  "spaceLength": 10,
-  "pointradius": 2,
-  "legend": {
-    "alignAsTable": true,
-    "avg": true,
-    "current": true,
-    "max": true,
-    "min": false,
-    "rightSide": true,
-    "show": true,
-    "sort": "current",
-    "sortDesc": true,
-    "total": false,
-    "values": true
-  },
-  "nullPointMode": "connected",
-  "tooltip": {
-    "value_type": "individual",
-    "shared": true,
-    "sort": 2
-  },
-  "aliasColors": {},
-  "seriesOverrides": [],
-  "thresholds": [],
-  "timeRegions": [],
-  "fill": 0,
-  "fillGradient": 0,
-  "dashes": false,
-  "hiddenSeries": false,
-  "points": false,
-  "bars": false,
-  "stack": false,
-  "percentage": false,
-  "steppedLine": false,
   "timeFrom": null,
-  "timeShift": null
+  "timeShift": null,
+  "interval": "30s"
 }

salt/grafana/panels/cpu_usage_tasks_zombies_graph.json.jinja

@@ -1,132 +1,124 @@
 {
-  "type": "graph",
+  "id": 69004,
-  "title": "CPU Tasks Zombies",
   "gridPos": {
     "x": {{ PANELS.cpu_usage_tasks_zombies_graph.gridPos.x }},
     "y": {{ PANELS.cpu_usage_tasks_zombies_graph.gridPos.y }},
     "w": {{ PANELS.cpu_usage_tasks_zombies_graph.gridPos.w }},
     "h": {{ PANELS.cpu_usage_tasks_zombies_graph.gridPos.h }}
   },
-  "id": 69004,
+  "type": "timeseries",
+  "title": "CPU Tasks Zombies",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 0,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "short",
+      "min": 0,
+      "decimals": 1
+    },
+    "overrides": []
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
   "targets": [
     {
-      "refId": "A",
+      "alias": "$tag_host $tag_role",
-      "queryType": "randomWalk",
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
       "groupBy": [
         {
-          "type": "time",
           "params": [
             "$__interval"
-          ]
+          ],
+          "type": "time"
         },
         {
-          "type": "fill",
           "params": [
             "null"
-          ]
+          ],
+          "type": "fill"
         }
       ],
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT mean(zombies) as zombies FROM \"processes\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), host, role ORDER BY asc",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "A",
+      "resultFormat": "time_series",
       "select": [
         [
           {
-            "type": "field",
             "params": [
               "value"
-            ]
+            ],
+            "type": "field"
           },
           {
-            "type": "mean",
+            "params": [],
-            "params": []
+            "type": "mean"
           }
         ]
       ],
-      "query": "SELECT mean(zombies) as zombies FROM \"processes\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), host, role ORDER BY asc",
+      "tags": []
-      "rawQuery": true,
-      "alias": "$tag_host $tag_role"
     }
   ],
-  "options": {
-    "alertThreshold": true
-  },
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
-  "pluginVersion": "7.5.4",
-  "renderer": "flot",
-  "yaxes": [
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": 0,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:412"
-    },
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": null,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:413"
-    }
-  ],
-  "xaxis": {
-    "show": true,
-    "mode": "time",
-    "name": null,
-    "values": [],
-    "buckets": null
-  },
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "lines": true,
-  "linewidth": 1,
-  "dashLength": 10,
-  "spaceLength": 10,
-  "pointradius": 2,
-  "legend": {
-    "alignAsTable": true,
-    "avg": true,
-    "current": true,
-    "max": true,
-    "min": false,
-    "rightSide": true,
-    "show": true,
-    "sort": "current",
-    "sortDesc": true,
-    "total": false,
-    "values": true
-  },
-  "nullPointMode": "connected",
-  "tooltip": {
-    "value_type": "individual",
-    "shared": true,
-    "sort": 2
-  },
-  "aliasColors": {},
-  "seriesOverrides": [],
-  "thresholds": [],
-  "timeRegions": [],
-  "fill": 0,
-  "fillGradient": 0,
-  "dashes": false,
-  "hiddenSeries": false,
-  "points": false,
-  "bars": false,
-  "stack": false,
-  "percentage": false,
-  "steppedLine": false,
   "timeFrom": null,
-  "timeShift": null
+  "timeShift": null,
+  "interval": "30s"
 }

salt/grafana/panels/disk_io_bytes_graph.json.jinja

@@ -1,194 +1,189 @@
 {
-    "aliasColors": {},
+  "id": 60200,
-    "maxDataPoints": 750,
+  "gridPos": {
-    "interval": "30s",
+    "x": {{ PANELS.disk_io_bytes_graph.gridPos.x }},
-    "bars": false,
+    "y": {{ PANELS.disk_io_bytes_graph.gridPos.y }},
-    "dashLength": 10,
+    "w": {{ PANELS.disk_io_bytes_graph.gridPos.w }},
-    "dashes": false,
+    "h": {{ PANELS.disk_io_bytes_graph.gridPos.h }}
-    "datasource": "InfluxDB",
+  },
-    "editable": true,
+  "type": "timeseries",
-    "error": false,
+  "title": "Disk I/O bytes for /dev/$disk",
-    "fieldConfig": {
+  "datasource": "InfluxDB",
-        "defaults": {
+  "pluginVersion": "8.2.1",
-            "links": []
+  "interval": "30s",
+  "links": [],
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 10,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
         },
-        "overrides": []
+        "axisPlacement": "auto",
-    },
+        "axisLabel": "",
-    "fill": 1,
+        "scaleDistribution": {
-    "fillGradient": 0,
+          "type": "linear"
-    "grid": {},
-    "gridPos": {
-        "x": {{ PANELS.disk_io_bytes_graph.gridPos.x }},
-        "y": {{ PANELS.disk_io_bytes_graph.gridPos.y }},
-        "w": {{ PANELS.disk_io_bytes_graph.gridPos.w }},
-        "h": {{ PANELS.disk_io_bytes_graph.gridPos.h }}
-    },
-    "hiddenSeries": false,
-    "id": 60200,
-    "legend": {
-        "alignAsTable": true,
-        "avg": true,
-        "current": true,
-        "hideEmpty": true,
-        "max": true,
-        "min": false,
-        "rightSide": false,
-        "show": true,
-        "sort": "current",
-        "sortDesc": true,
-        "total": false,
-        "values": true
-    },
-    "lines": true,
-    "linewidth": 1,
-    "links": [],
-    "maxPerRow": 6,
-    "nullPointMode": "connected",
-    "options": {
-        "alertThreshold": true
-    },
-    "percentage": false,
-    "pluginVersion": "7.5.4",
-    "pointradius": 5,
-    "points": false,
-    "renderer": "flot",
-    "repeat": null,
-
-    "seriesOverrides": [],
-    "spaceLength": 10,
-    "stack": false,
-    "steppedLine": false,
-    "targets": [{
-            "alias": "$tag_host: $tag_name: $col",
-            "dsType": "influxdb",
-            "function": "mean",
-            "groupBy": [{
-                    "interval": "auto",
-                    "params": [
-                        "auto"
-                    ],
-                    "type": "time"
-                },
-                {
-                    "key": "host",
-                    "params": [
-                        "tag"
-                    ],
-                    "type": "tag"
-                },
-                {
-                    "key": "path",
-                    "params": [
-                        "tag"
-                    ],
-                    "type": "tag"
-                }
-            ],
-            "measurement": "io_reads",
-            "policy": "default",
-            "query": "SELECT non_negative_derivative(mean(read_bytes),1s) as \"read\" FROM \"diskio\" WHERE \"host\" =~ /$servername$/ AND \"name\" =~ /$disk$/  AND $timeFilter GROUP BY time($__interval), *",
-            "rawQuery": true,
-            "refId": "B",
-            "resultFormat": "time_series",
-            "select": [
-                [{
-                        "params": [
-                            "value"
-                        ],
-                        "type": "field"
-                    },
-                    {
-                        "params": [],
-                        "type": "mean"
-                    }
-                ]
-            ],
-            "tags": []
         },
-        {
+        "hideFrom": {
-            "alias": "$tag_host: $tag_name: $col",
+          "tooltip": false,
-            "dsType": "influxdb",
+          "viz": false,
-            "function": "mean",
+          "legend": false
-            "groupBy": [{
+        },
-                    "interval": "auto",
+        "thresholdsStyle": {
-                    "params": [
+          "mode": "off"
-                        "auto"
-                    ],
-                    "type": "time"
-                },
-                {
-                    "key": "host",
-                    "params": [
-                        "tag"
-                    ],
-                    "type": "tag"
-                },
-                {
-                    "key": "path",
-                    "params": [
-                        "tag"
-                    ],
-                    "type": "tag"
-                }
-            ],
-            "measurement": "io_reads",
-            "policy": "default",
-            "query": "SELECT non_negative_derivative(mean(write_bytes),1s) as \"write\" FROM \"diskio\" WHERE \"host\" =~ /$servername$/ AND \"name\" =~ /$disk$/ AND $timeFilter GROUP BY time($__interval), *",
-            "rawQuery": true,
-            "refId": "C",
-            "resultFormat": "time_series",
-            "select": [
-                [{
-                        "params": [
-                            "value"
-                        ],
-                        "type": "field"
-                    },
-                    {
-                        "params": [],
-                        "type": "mean"
-                    }
-                ]
-            ],
-            "tags": []
         }
-    ],
+      },
-    "thresholds": [],
+      "color": {
-    "timeFrom": null,
+        "mode": "palette-classic"
-    "timeRegions": [],
+      },
-    "timeShift": null,
+      "thresholds": {
-    "title": "Disk I/O bytes for /dev/$disk",
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "links": [],
+      "unit": "bytes",
+      "decimals": 1
+    },
+    "overrides": []
+  },
+  "options": {
     "tooltip": {
-        "msResolution": false,
+      "mode": "single"
-        "shared": true,
-        "sort": 0,
-        "value_type": "cumulative"
     },
-    "type": "graph",
+    "legend": {
-    "xaxis": {
+      "displayMode": "table",
-        "buckets": null,
+      "placement": "bottom",
-        "mode": "time",
+      "calcs": [
-        "name": null,
+        "max",
-        "show": true,
+        "mean",
-        "values": []
+        "lastNotNull"
-    },
+      ]
-    "yaxes": [{
+    }
-            "format": "bytes",
+  },
-            "logBase": 1,
+  "targets": [
-            "max": null,
+    {
-            "min": null,
+      "alias": "$tag_host: $tag_name: $col",
-            "show": true
+      "dsType": "influxdb",
+      "function": "mean",
+      "groupBy": [
+        {
+          "interval": "auto",
+          "params": [
+            "auto"
+          ],
+          "type": "time"
         },
         {
-            "format": "short",
+          "key": "host",
-            "logBase": 1,
+          "params": [
-            "max": null,
+            "tag"
-            "min": null,
+          ],
-            "show": true
+          "type": "tag"
+        },
+        {
+          "key": "path",
+          "params": [
+            "tag"
+          ],
+          "type": "tag"
         }
-    ],
+      ],
-    "yaxis": {
+      "measurement": "io_reads",
-        "align": false,
+      "policy": "default",
-        "alignLevel": null
+      "query": "SELECT non_negative_derivative(mean(read_bytes),1s) as \"read\" FROM \"diskio\" WHERE \"host\" =~ /$servername$/ AND \"name\" =~ /$disk$/  AND $timeFilter GROUP BY time($__interval), *",
+      "rawQuery": true,
+      "refId": "B",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "value"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
+      "tags": []
+    },
+    {
+      "alias": "$tag_host: $tag_name: $col",
+      "dsType": "influxdb",
+      "function": "mean",
+      "groupBy": [
+        {
+          "interval": "auto",
+          "params": [
+            "auto"
+          ],
+          "type": "time"
+        },
+        {
+          "key": "host",
+          "params": [
+            "tag"
+          ],
+          "type": "tag"
+        },
+        {
+          "key": "path",
+          "params": [
+            "tag"
+          ],
+          "type": "tag"
+        }
+      ],
+      "measurement": "io_reads",
+      "policy": "default",
+      "query": "SELECT non_negative_derivative(mean(write_bytes),1s) as \"write\" FROM \"diskio\" WHERE \"host\" =~ /$servername$/ AND \"name\" =~ /$disk$/ AND $timeFilter GROUP BY time($__interval), *",
+      "rawQuery": true,
+      "refId": "C",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "value"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
+      "tags": []
     }
+  ],
+  "scopedVars": {
+    "disk": {
+      "text": "sda",
+      "value": "sda",
+      "selected": false
+    }
+  },
+  "maxDataPoints": null,
+  "repeat": null,
+  "timeFrom": null,
+  "timeShift": null
 }

salt/grafana/panels/disk_io_requests_graph.json.jinja

@@ -1,193 +1,190 @@
 {
-    "aliasColors": {},
+  "id": 13782,
-    "maxDataPoints": 750,
-    "interval": "30s",
-    "bars": false,
-    "dashLength": 10,
-    "dashes": false,
-    "datasource": "InfluxDB",
-    "editable": true,
-    "error": false,
-    "fieldConfig": {
-        "defaults": {
-            "links": []
-        },
-        "overrides": []
-    },
-    "fill": 1,
-    "fillGradient": 0,
-    "grid": {},
   "gridPos": {
     "x": {{ PANELS.disk_io_requests_graph.gridPos.x }},
     "y": {{ PANELS.disk_io_requests_graph.gridPos.y }},
     "w": {{ PANELS.disk_io_requests_graph.gridPos.w }},
     "h": {{ PANELS.disk_io_requests_graph.gridPos.h }}
   },
-    "hiddenSeries": false,
+  "type": "timeseries",
-    "id": 13782,
+  "title": "Disk I/O requests for /dev/$disk",
-    "legend": {
+  "datasource": "InfluxDB",
-        "alignAsTable": true,
+  "pluginVersion": "8.2.1",
-        "avg": true,
+  "interval": "30s",
-        "current": true,
+  "links": [],
-        "hideEmpty": true,
+  "fieldConfig": {
-        "max": true,
+    "defaults": {
-        "min": false,
+      "custom": {
-        "rightSide": false,
+        "drawStyle": "line",
-        "show": true,
+        "lineInterpolation": "linear",
-        "sort": "current",
+        "barAlignment": 0,
-        "sortDesc": true,
+        "lineWidth": 1,
-        "total": false,
+        "fillOpacity": 10,
-        "values": true
+        "gradientMode": "none",
-    },
+        "spanNulls": false,
-    "lines": true,
+        "showPoints": "never",
-    "linewidth": 1,
+        "pointSize": 5,
-    "links": [],
+        "stacking": {
-    "maxPerRow": 6,
+          "mode": "none",
-    "nullPointMode": "connected",
+          "group": "A"
-    "options": {
-        "alertThreshold": true
-    },
-    "percentage": false,
-    "pluginVersion": "7.5.4",
-    "pointradius": 5,
-    "points": false,
-    "renderer": "flot",
-    "repeat": null,
-    "seriesOverrides": [],
-    "spaceLength": 10,
-    "stack": false,
-    "steppedLine": false,
-    "targets": [{
-            "alias": "$tag_host: $tag_name: $col",
-            "dsType": "influxdb",
-            "function": "mean",
-            "groupBy": [{
-                    "interval": "auto",
-                    "params": [
-                        "auto"
-                    ],
-                    "type": "time"
-                },
-                {
-                    "key": "host",
-                    "params": [
-                        "tag"
-                    ],
-                    "type": "tag"
-                },
-                {
-                    "key": "path",
-                    "params": [
-                        "tag"
-                    ],
-                    "type": "tag"
-                }
-            ],
-            "measurement": "io_reads",
-            "policy": "default",
-            "query": "SELECT non_negative_derivative(mean(reads),1s) as \"read\" FROM \"diskio\" WHERE \"host\" =~ /$servername$/ AND \"name\" =~ /$disk$/ AND $timeFilter GROUP BY time($__interval), *",
-            "rawQuery": true,
-            "refId": "B",
-            "resultFormat": "time_series",
-            "select": [
-                [{
-                        "params": [
-                            "value"
-                        ],
-                        "type": "field"
-                    },
-                    {
-                        "params": [],
-                        "type": "mean"
-                    }
-                ]
-            ],
-            "tags": []
         },
-        {
+        "axisPlacement": "auto",
-            "alias": "$tag_host: $tag_name: $col",
+        "axisLabel": "",
-            "dsType": "influxdb",
+        "scaleDistribution": {
-            "function": "mean",
+          "type": "linear"
-            "groupBy": [{
+        },
-                    "interval": "auto",
+        "hideFrom": {
-                    "params": [
+          "tooltip": false,
-                        "auto"
+          "viz": false,
-                    ],
+          "legend": false
-                    "type": "time"
+        },
-                },
+        "thresholdsStyle": {
-                {
+          "mode": "off"
-                    "key": "host",
-                    "params": [
-                        "tag"
-                    ],
-                    "type": "tag"
-                },
-                {
-                    "key": "path",
-                    "params": [
-                        "tag"
-                    ],
-                    "type": "tag"
-                }
-            ],
-            "measurement": "io_reads",
-            "policy": "default",
-            "query": "SELECT non_negative_derivative(mean(writes),1s) as \"write\" FROM \"diskio\" WHERE \"host\" =~ /$servername$/ AND \"name\" =~ /$disk$/  AND $timeFilter GROUP BY time($__interval), *",
-            "rawQuery": true,
-            "refId": "C",
-            "resultFormat": "time_series",
-            "select": [
-                [{
-                        "params": [
-                            "value"
-                        ],
-                        "type": "field"
-                    },
-                    {
-                        "params": [],
-                        "type": "mean"
-                    }
-                ]
-            ],
-            "tags": []
         }
-    ],
+      },
-    "thresholds": [],
+      "color": {
-    "timeFrom": null,
+        "mode": "palette-classic"
-    "timeRegions": [],
+      },
-    "timeShift": null,
+      "thresholds": {
-    "title": "Disk I/O requests for /dev/$disk",
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "links": [],
+      "unit": "iops",
+      "decimals": 1
+    },
+    "overrides": []
+  },
+  "options": {
     "tooltip": {
-        "msResolution": false,
+      "mode": "single"
-        "shared": true,
-        "sort": 0,
-        "value_type": "cumulative"
     },
-    "type": "graph",
+    "legend": {
-    "xaxis": {
+      "displayMode": "table",
-        "buckets": null,
+      "placement": "bottom",
-        "mode": "time",
+      "calcs": [
-        "name": null,
+        "max",
-        "show": true,
+        "mean",
-        "values": []
+        "lastNotNull"
-    },
+      ]
-    "yaxes": [{
+    }
-            "format": "iops",
+  },
-            "logBase": 1,
+  "targets": [
-            "max": null,
+    {
-            "min": null,
+      "alias": "$tag_host: $tag_name: $col",
-            "show": true
+      "dsType": "influxdb",
+      "function": "mean",
+      "groupBy": [
+        {
+          "interval": "auto",
+          "params": [
+            "auto"
+          ],
+          "type": "time"
         },
         {
-            "format": "short",
+          "key": "host",
-            "logBase": 1,
+          "params": [
-            "max": null,
+            "tag"
-            "min": null,
+          ],
-            "show": true
+          "type": "tag"
+        },
+        {
+          "key": "path",
+          "params": [
+            "tag"
+          ],
+          "type": "tag"
         }
-    ],
+      ],
-    "yaxis": {
+      "measurement": "io_reads",
-        "align": false,
+      "policy": "default",
-        "alignLevel": null
+      "query": "SELECT non_negative_derivative(mean(reads),1s) as \"read\" FROM \"diskio\" WHERE \"host\" =~ /$servername$/ AND \"name\" =~ /$disk$/ AND $timeFilter GROUP BY time($__interval), *",
+      "rawQuery": true,
+      "refId": "B",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "value"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
+      "tags": [],
+      "hide": false
+    },
+    {
+      "alias": "$tag_host: $tag_name: $col",
+      "dsType": "influxdb",
+      "function": "mean",
+      "groupBy": [
+        {
+          "interval": "auto",
+          "params": [
+            "auto"
+          ],
+          "type": "time"
+        },
+        {
+          "key": "host",
+          "params": [
+            "tag"
+          ],
+          "type": "tag"
+        },
+        {
+          "key": "path",
+          "params": [
+            "tag"
+          ],
+          "type": "tag"
+        }
+      ],
+      "measurement": "io_reads",
+      "policy": "default",
+      "query": "SELECT non_negative_derivative(mean(writes),1s) as \"write\" FROM \"diskio\" WHERE \"host\" =~ /$servername$/ AND \"name\" =~ /$disk$/  AND $timeFilter GROUP BY time($__interval), *",
+      "rawQuery": true,
+      "refId": "C",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "value"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
+      "tags": []
     }
+  ],
+  "scopedVars": {
+    "disk": {
+      "text": "sda",
+      "value": "sda",
+      "selected": false
+    }
+  },
+  "maxDataPoints": null,
+  "repeat": null,
+  "timeFrom": null,
+  "timeShift": null
 }

salt/grafana/panels/disk_io_time_graph.json.jinja

@@ -1,193 +1,189 @@
 {
-    "aliasColors": {},
+  "id": 56720,
-    "maxDataPoints": 750,
-    "interval": "30s",
-    "bars": false,
-    "dashLength": 10,
-    "dashes": false,
-    "datasource": "InfluxDB",
-    "editable": true,
-    "error": false,
-    "fieldConfig": {
-        "defaults": {
-            "links": []
-        },
-        "overrides": []
-    },
-    "fill": 1,
-    "fillGradient": 0,
-    "grid": {},
   "gridPos": {
     "x": {{ PANELS.disk_io_time_graph.gridPos.x }},
     "y": {{ PANELS.disk_io_time_graph.gridPos.y }},
     "w": {{ PANELS.disk_io_time_graph.gridPos.w }},
     "h": {{ PANELS.disk_io_time_graph.gridPos.h }}
   },
-    "hiddenSeries": false,
+  "type": "timeseries",
-    "id": 56720,
+  "title": "Disk I/O time for /dev/$disk",
-    "legend": {
+  "datasource": "InfluxDB",
-        "alignAsTable": true,
+  "pluginVersion": "8.2.1",
-        "avg": true,
+  "interval": "30s",
-        "current": true,
+  "links": [],
-        "hideEmpty": true,
+  "fieldConfig": {
-        "max": true,
+    "defaults": {
-        "min": false,
+      "custom": {
-        "rightSide": false,
+        "drawStyle": "line",
-        "show": true,
+        "lineInterpolation": "linear",
-        "sort": "current",
+        "barAlignment": 0,
-        "sortDesc": true,
+        "lineWidth": 1,
-        "total": false,
+        "fillOpacity": 10,
-        "values": true
+        "gradientMode": "none",
-    },
+        "spanNulls": false,
-    "lines": true,
+        "showPoints": "never",
-    "linewidth": 1,
+        "pointSize": 5,
-    "links": [],
+        "stacking": {
-    "maxPerRow": 6,
+          "mode": "none",
-    "nullPointMode": "connected",
+          "group": "A"
-    "options": {
-        "alertThreshold": true
-    },
-    "percentage": false,
-    "pluginVersion": "7.5.4",
-    "pointradius": 5,
-    "points": false,
-    "renderer": "flot",
-    "repeat": null,
-    "seriesOverrides": [],
-    "spaceLength": 10,
-    "stack": false,
-    "steppedLine": false,
-    "targets": [{
-            "alias": "$tag_host: $tag_name: $col",
-            "dsType": "influxdb",
-            "function": "mean",
-            "groupBy": [{
-                    "interval": "auto",
-                    "params": [
-                        "auto"
-                    ],
-                    "type": "time"
-                },
-                {
-                    "key": "host",
-                    "params": [
-                        "tag"
-                    ],
-                    "type": "tag"
-                },
-                {
-                    "key": "path",
-                    "params": [
-                        "tag"
-                    ],
-                    "type": "tag"
-                }
-            ],
-            "measurement": "io_reads",
-            "policy": "default",
-            "query": "SELECT non_negative_derivative(mean(read_time),1s) as \"read\" FROM \"diskio\" WHERE \"host\" =~ /$servername$/ AND \"name\" =~ /$disk$/  AND $timeFilter GROUP BY time($__interval), *",
-            "rawQuery": true,
-            "refId": "B",
-            "resultFormat": "time_series",
-            "select": [
-                [{
-                        "params": [
-                            "value"
-                        ],
-                        "type": "field"
-                    },
-                    {
-                        "params": [],
-                        "type": "mean"
-                    }
-                ]
-            ],
-            "tags": []
         },
-        {
+        "axisPlacement": "auto",
-            "alias": "$tag_host: $tag_name: $col",
+        "axisLabel": "",
-            "dsType": "influxdb",
+        "scaleDistribution": {
-            "function": "mean",
+          "type": "linear"
-            "groupBy": [{
+        },
-                    "interval": "auto",
+        "hideFrom": {
-                    "params": [
+          "tooltip": false,
-                        "auto"
+          "viz": false,
-                    ],
+          "legend": false
-                    "type": "time"
+        },
-                },
+        "thresholdsStyle": {
-                {
+          "mode": "off"
-                    "key": "host",
-                    "params": [
-                        "tag"
-                    ],
-                    "type": "tag"
-                },
-                {
-                    "key": "path",
-                    "params": [
-                        "tag"
-                    ],
-                    "type": "tag"
-                }
-            ],
-            "measurement": "io_reads",
-            "policy": "default",
-            "query": "SELECT non_negative_derivative(mean(write_time),1s) as \"write\" FROM \"diskio\" WHERE \"host\" =~ /$servername$/ AND \"name\" =~ /$disk$/ AND $timeFilter GROUP BY time($__interval), *",
-            "rawQuery": true,
-            "refId": "A",
-            "resultFormat": "time_series",
-            "select": [
-                [{
-                        "params": [
-                            "value"
-                        ],
-                        "type": "field"
-                    },
-                    {
-                        "params": [],
-                        "type": "mean"
-                    }
-                ]
-            ],
-            "tags": []
         }
-    ],
+      },
-    "thresholds": [],
+      "color": {
-    "timeFrom": null,
+        "mode": "palette-classic"
-    "timeRegions": [],
+      },
-    "timeShift": null,
+      "thresholds": {
-    "title": "Disk I/O time for /dev/$disk",
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "links": [],
+      "unit": "ms",
+      "decimals": 1
+    },
+    "overrides": []
+  },
+  "options": {
     "tooltip": {
-        "msResolution": false,
+      "mode": "single"
-        "shared": true,
-        "sort": 0,
-        "value_type": "cumulative"
     },
-    "type": "graph",
+    "legend": {
-    "xaxis": {
+      "displayMode": "table",
-        "buckets": null,
+      "placement": "bottom",
-        "mode": "time",
+      "calcs": [
-        "name": null,
+        "max",
-        "show": true,
+        "mean",
-        "values": []
+        "lastNotNull"
-    },
+      ]
-    "yaxes": [{
+    }
-            "format": "ms",
+  },
-            "logBase": 1,
+  "targets": [
-            "max": null,
+    {
-            "min": null,
+      "alias": "$tag_host: $tag_name: $col",
-            "show": true
+      "dsType": "influxdb",
+      "function": "mean",
+      "groupBy": [
+        {
+          "interval": "auto",
+          "params": [
+            "auto"
+          ],
+          "type": "time"
         },
         {
-            "format": "short",
+          "key": "host",
-            "logBase": 1,
+          "params": [
-            "max": null,
+            "tag"
-            "min": null,
+          ],
-            "show": true
+          "type": "tag"
+        },
+        {
+          "key": "path",
+          "params": [
+            "tag"
+          ],
+          "type": "tag"
         }
-    ],
+      ],
-    "yaxis": {
+      "measurement": "io_reads",
-        "align": false,
+      "policy": "default",
-        "alignLevel": null
+      "query": "SELECT non_negative_derivative(mean(read_time),1s) as \"read\" FROM \"diskio\" WHERE \"host\" =~ /$servername$/ AND \"name\" =~ /$disk$/  AND $timeFilter GROUP BY time($__interval), *",
+      "rawQuery": true,
+      "refId": "B",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "value"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
+      "tags": []
+    },
+    {
+      "alias": "$tag_host: $tag_name: $col",
+      "dsType": "influxdb",
+      "function": "mean",
+      "groupBy": [
+        {
+          "interval": "auto",
+          "params": [
+            "auto"
+          ],
+          "type": "time"
+        },
+        {
+          "key": "host",
+          "params": [
+            "tag"
+          ],
+          "type": "tag"
+        },
+        {
+          "key": "path",
+          "params": [
+            "tag"
+          ],
+          "type": "tag"
+        }
+      ],
+      "measurement": "io_reads",
+      "policy": "default",
+      "query": "SELECT non_negative_derivative(mean(write_time),1s) as \"write\" FROM \"diskio\" WHERE \"host\" =~ /$servername$/ AND \"name\" =~ /$disk$/ AND $timeFilter GROUP BY time($__interval), *",
+      "rawQuery": true,
+      "refId": "A",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "value"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
+      "tags": []
     }
+  ],
+  "scopedVars": {
+    "disk": {
+      "text": "sda",
+      "value": "sda",
+      "selected": false
+    }
+  },
+  "maxDataPoints": null,
+  "repeat": null,
+  "timeFrom": null,
+  "timeShift": null
 }

salt/grafana/panels/disk_usage_nsm_graph.json.jinja

@@ -1,186 +1,211 @@
 {
-  "type": "graph",
+  "id": 68888,
-  "title": "Disk Usage /nsm",
   "gridPos": {
     "x": {{ PANELS.disk_usage_nsm_graph.gridPos.x }},
     "y": {{ PANELS.disk_usage_nsm_graph.gridPos.y }},
     "w": {{ PANELS.disk_usage_nsm_graph.gridPos.w }},
     "h": {{ PANELS.disk_usage_nsm_graph.gridPos.h }}
   },
-  "id": 68888,
+  "type": "timeseries",
+  "title": "Disk Usage /nsm",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "bottom",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
   "targets": [
     {
-      "refId": "A",
-      "queryType": "randomWalk",
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
-      "groupBy": [
-        {
-          "type": "time",
-          "params": [
-            "$__interval"
-          ]
-        },
-        {
-          "type": "fill",
-          "params": [
-            "null"
-          ]
-        }
-      ],
-      "select": [
-        [
-          {
-            "type": "field",
-            "params": [
-              "value"
-            ]
-          },
-          {
-            "type": "mean",
-            "params": []
-          }
-        ]
-      ],
-      "query": "SELECT mean(total) AS \"total\", mean(used) as \"used\" FROM \"disk\" WHERE \"host\" =~ /$servername$/ AND \"path\" = '/nsm' AND $timeFilter GROUP BY time($__interval), \"host\", \"path\"",
-      "rawQuery": true,
-      "alias": "$tag_host: mountpoint $tag_path - $col"
-    },
-    {
-      "refId": "B",
-      "queryType": "randomWalk",
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
-      "groupBy": [
-        {
-          "type": "time",
-          "params": [
-            "$__interval"
-          ]
-        },
-        {
-          "type": "fill",
-          "params": [
-            "null"
-          ]
-        }
-      ],
-      "select": [
-        [
-          {
-            "type": "field",
-            "params": [
-              "value"
-            ]
-          },
-          {
-            "type": "mean",
-            "params": []
-          }
-        ]
-      ],
-      "query": "SELECT mean(mean_total) AS \"trend_total\", mean(mean_used) as \"trend_used\" FROM \"so_long_term\".\"disk\" WHERE \"host\" =~ /$servername$/ AND \"path\" = '/nsm' AND $timeFilter GROUP BY time($__interval), \"host\", \"path\"",
-      "rawQuery": true,
       "alias": "$tag_host: mountpoint $tag_path - $col",
-      "hide": false
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT mean(total) AS \"total\", mean(used) as \"used\" FROM \"disk\" WHERE \"host\" =~ /$servername$/ AND \"path\" = '/nsm' AND $timeFilter GROUP BY time($__interval), \"host\", \"path\"",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "A",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "value"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
+      "tags": []
+    },
+    {
+      "alias": "$tag_host: mountpoint $tag_path - $col",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "hide": false,
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT mean(mean_total) AS \"trend_total\", mean(mean_used) as \"trend_used\" FROM \"so_long_term\".\"disk\" WHERE \"host\" =~ /$servername$/ AND \"path\" = '/nsm' AND $timeFilter GROUP BY time($__interval), \"host\", \"path\" fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "B",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "value"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
+      "tags": []
     }
   ],
-  "options": {
-    "alertThreshold": true
-  },
-  "datasource": "InfluxDB",
   "fieldConfig": {
-    "defaults": {},
+    "defaults": {
-    "overrides": []
+      "custom": {
-  },
+        "drawStyle": "line",
-  "pluginVersion": "7.5.4",
+        "lineInterpolation": "linear",
-  "renderer": "flot",
+        "barAlignment": 0,
-  "yaxes": [
+        "lineWidth": 1,
-    {
+        "fillOpacity": 10,
-      "label": null,
+        "gradientMode": "none",
-      "show": true,
+        "spanNulls": false,
-      "logBase": 1,
+        "showPoints": "never",
-      "min": "0",
+        "pointSize": 5,
-      "max": null,
+        "stacking": {
-      "format": "bytes",
+          "mode": "none",
-      "$$hashKey": "object:235"
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "bytes",
+      "min": 0,
+      "decimals": 1
     },
-    {
+    "overrides": [
-      "label": null,
+      {
-      "show": true,
+        "matcher": {
-      "logBase": 1,
+          "id": "byRegexp",
-      "min": null,
+          "options": "/total/"
-      "max": null,
+        },
-      "format": "short",
+        "properties": [
-      "$$hashKey": "object:236"
+          {
-    }
+            "id": "color",
-  ],
+            "value": {
-  "xaxis": {
+              "fixedColor": "#C4162A",
-    "show": true,
+              "mode": "fixed"
-    "mode": "time",
+            }
-    "name": null,
+          },
-    "values": [],
+          {
-    "buckets": null
+            "id": "custom.fillOpacity",
+            "value": 0
+          },
+          {
+            "id": "custom.lineWidth",
+            "value": 2
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/trend/"
+        },
+        "properties": [
+          {
+            "id": "custom.fillOpacity",
+            "value": 0
+          },
+          {
+            "id": "custom.lineWidth",
+            "value": 4
+          },
+          {
+            "id": "custom.lineStyle",
+            "value": {
+              "fill": "dash",
+              "dash": [
+                4,
+                10
+              ]
+            }
+          }
+        ]
+      }
+    ]
   },
-  "yaxis": {
+  "maxDataPoints": null,
-    "align": false,
+  "timeFrom": null,
-    "alignLevel": null
+  "timeShift": null
-  },
-  "lines": true,
-  "fill": 1,
-  "linewidth": 1,
-  "dashLength": 10,
-  "spaceLength": 10,
-  "pointradius": 2,
-  "legend": {
-    "show": true,
-    "values": true,
-    "min": false,
-    "max": true,
-    "current": true,
-    "total": false,
-    "avg": true,
-    "alignAsTable": true
-  },
-  "nullPointMode": "connected",
-  "tooltip": {
-    "value_type": "individual",
-    "shared": true,
-    "sort": 0
-  },
-  "aliasColors": {},
-  "seriesOverrides": [
-    {
-      "$$hashKey": "object:486",
-      "alias": "/total/",
-      "fill": 0,
-      "linewidth": 2,
-      "color": "#C4162A",
-      "zindex": 3
-    },
-    {
-      "$$hashKey": "object:829",
-      "alias": "/trend/",
-      "fill": 0,
-      "linewidth": 4,
-      "dashes": true,
-      "dashLength": 4
-    }
-  ],
-  "thresholds": [],
-  "timeRegions": [],
-  "steppedLine": true,
-  "fillGradient": 0,
-  "dashes": false,
-  "hiddenSeries": false,
-  "points": false,
-  "bars": false,
-  "stack": false,
-  "percentage": false,
-  "maxDataPoints": 750,
-  "interval": "30s"
 }

salt/grafana/panels/disk_usage_nsm_percent_graph.json.jinja

@@ -1,45 +1,82 @@
 {
-  "aliasColors": {},
+  "id": 47230,
-  "dashLength": 10,
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
   "gridPos": {
     "x": {{ PANELS.disk_usage_nsm_percent_graph.gridPos.x }},
     "y": {{ PANELS.disk_usage_nsm_percent_graph.gridPos.y }},
     "w": {{ PANELS.disk_usage_nsm_percent_graph.gridPos.w }},
     "h": {{ PANELS.disk_usage_nsm_percent_graph.gridPos.h }}
   },
-  "id": 47230,
+  "type": "timeseries",
+  "title": "Disk Usage /nsm",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
   "interval": "30s",
-  "legend": {
+  "fieldConfig": {
-    "alignAsTable": true,
+    "defaults": {
-    "avg": false,
+      "custom": {
-    "current": true,
+        "drawStyle": "line",
-    "max": false,
+        "lineInterpolation": "linear",
-    "min": false,
+        "barAlignment": 0,
-    "rightSide": true,
+        "lineWidth": 1,
-    "show": true,
+        "fillOpacity": 0,
-    "sort": "current",
+        "gradientMode": "none",
-    "sortDesc": true,
+        "spanNulls": false,
-    "total": false,
+        "showPoints": "never",
-    "values": true
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "percent",
+      "decimals": 1,
+      "min": 0,
+      "max": 100
+    },
+    "overrides": []
   },
-  "lines": true,
-  "linewidth": 1,
-  "maxDataPoints": 750,
-  "nullPointMode": "connected",
   "options": {
-    "alertThreshold": false
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "lastNotNull"
+      ]
+    }
   },
-  "pluginVersion": "7.5.4",
-  "pointradius": 2,
-  "renderer": "flot",
-  "seriesOverrides": [],
-  "spaceLength": 10,
-  "steppedLine": true,
   "targets": [
     {
       "alias": "$tag_host $tag_role",
@@ -51,16 +88,16 @@
           "type": "time"
         },
         {
-          "type": "tag",
           "params": [
             "host"
-          ]
+          ],
+          "type": "tag"
         },
         {
-          "type": "tag",
           "params": [
             "role"
-          ]
+          ],
+          "type": "tag"
         },
         {
           "params": [
@@ -69,6 +106,7 @@
           "type": "fill"
         }
       ],
+      "measurement": "disk",
       "orderByTime": "ASC",
       "policy": "default",
       "query": "SELECT mean(total) AS \"total\", mean(used) as \"used\" FROM \"disk\" WHERE \"host\" =~ /$servername$/ AND \"path\" = '/nsm' AND $timeFilter GROUP BY time($__interval), \"host\", \"path\"",
@@ -102,60 +140,10 @@
           "operator": "=",
           "value": "/nsm"
         }
-      ],
+      ]
-      "measurement": "disk"
     }
   ],
-  "thresholds": [],
+  "maxDataPoints": null,
-  "timeRegions": [],
-  "title": "Disk Usage /nsm",
-  "tooltip": {
-    "shared": true,
-    "sort": 2,
-    "value_type": "individual"
-  },
-  "type": "graph",
-  "xaxis": {
-    "buckets": null,
-    "mode": "time",
-    "name": null,
-    "show": true,
-    "values": []
-  },
-  "yaxes": [
-    {
-      "$$hashKey": "object:235",
-      "format": "percent",
-      "label": "",
-      "logBase": 1,
-      "max": "100",
-      "min": "0",
-      "show": true,
-      "decimals": 1
-    },
-    {
-      "$$hashKey": "object:236",
-      "format": "short",
-      "label": null,
-      "logBase": 1,
-      "max": null,
-      "min": null,
-      "show": true
-    }
-  ],
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "fill": 0,
-  "bars": false,
-  "dashes": false,
-  "fillGradient": 0,
-  "hiddenSeries": false,
-  "percentage": false,
-  "points": false,
-  "stack": false,
   "timeFrom": null,
-  "timeShift": null,
+  "timeShift": null
-  "decimals": 1
 }

salt/grafana/panels/disk_usage_root_graph.json.jinja

@@ -1,186 +1,211 @@
 {
-  "type": "graph",
+  "id": 61880,
-  "title": "Disk Usage /",
   "gridPos": {
     "x": {{ PANELS.disk_usage_root_graph.gridPos.x }},
     "y": {{ PANELS.disk_usage_root_graph.gridPos.y }},
     "w": {{ PANELS.disk_usage_root_graph.gridPos.w }},
     "h": {{ PANELS.disk_usage_root_graph.gridPos.h }}
   },
-  "id": 61880,
+  "type": "timeseries",
+  "title": "Disk Usage /",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "bottom",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
   "targets": [
     {
-      "refId": "A",
-      "queryType": "randomWalk",
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
-      "groupBy": [
-        {
-          "type": "time",
-          "params": [
-            "$__interval"
-          ]
-        },
-        {
-          "type": "fill",
-          "params": [
-            "null"
-          ]
-        }
-      ],
-      "select": [
-        [
-          {
-            "type": "field",
-            "params": [
-              "value"
-            ]
-          },
-          {
-            "type": "mean",
-            "params": []
-          }
-        ]
-      ],
-      "query": "SELECT mean(total) AS \"total\", mean(used) as \"used\" FROM \"disk\" WHERE \"host\" =~ /$servername$/ AND \"path\" = '/' AND $timeFilter GROUP BY time($__interval), \"host\", \"path\"",
-      "rawQuery": true,
-      "alias": "$tag_host: mountpoint $tag_path - $col"
-    },
-    {
-      "refId": "B",
-      "queryType": "randomWalk",
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
-      "groupBy": [
-        {
-          "type": "time",
-          "params": [
-            "$__interval"
-          ]
-        },
-        {
-          "type": "fill",
-          "params": [
-            "null"
-          ]
-        }
-      ],
-      "select": [
-        [
-          {
-            "type": "field",
-            "params": [
-              "value"
-            ]
-          },
-          {
-            "type": "mean",
-            "params": []
-          }
-        ]
-      ],
-      "query": "SELECT mean(mean_total) AS \"trend_total\", mean(mean_used) as \"trend_used\" FROM \"so_long_term\".\"disk\" WHERE \"host\" =~ /$servername$/ AND \"path\" = '/' AND $timeFilter GROUP BY time($__interval), \"host\", \"path\"",
-      "rawQuery": true,
       "alias": "$tag_host: mountpoint $tag_path - $col",
-      "hide": false
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT mean(total) AS \"total\", mean(used) as \"used\" FROM \"disk\" WHERE \"host\" =~ /$servername$/ AND \"path\" = '/' AND $timeFilter GROUP BY time($__interval), \"host\", \"path\"",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "A",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "value"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
+      "tags": []
+    },
+    {
+      "alias": "$tag_host: mountpoint $tag_path - $col",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "hide": false,
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT mean(mean_total) AS \"trend_total\", mean(mean_used) as \"trend_used\" FROM \"so_long_term\".\"disk\" WHERE \"host\" =~ /$servername$/ AND \"path\" = '/' AND $timeFilter GROUP BY time($__interval), \"host\", \"path\" fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "B",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "value"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
+      "tags": []
     }
   ],
-  "options": {
-    "alertThreshold": true
-  },
-  "datasource": "InfluxDB",
   "fieldConfig": {
-    "defaults": {},
+    "defaults": {
-    "overrides": []
+      "custom": {
-  },
+        "drawStyle": "line",
-  "pluginVersion": "7.5.4",
+        "lineInterpolation": "stepAfter",
-  "renderer": "flot",
+        "barAlignment": 0,
-  "yaxes": [
+        "lineWidth": 1,
-    {
+        "fillOpacity": 10,
-      "label": null,
+        "gradientMode": "none",
-      "show": true,
+        "spanNulls": false,
-      "logBase": 1,
+        "showPoints": "never",
-      "min": "0",
+        "pointSize": 5,
-      "max": null,
+        "stacking": {
-      "format": "bytes",
+          "mode": "none",
-      "$$hashKey": "object:235"
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "bytes",
+      "min": 0,
+      "decimals": 1
     },
-    {
+    "overrides": [
-      "label": null,
+      {
-      "show": true,
+        "matcher": {
-      "logBase": 1,
+          "id": "byRegexp",
-      "min": null,
+          "options": "/total/"
-      "max": null,
+        },
-      "format": "short",
+        "properties": [
-      "$$hashKey": "object:236"
+          {
-    }
+            "id": "color",
-  ],
+            "value": {
-  "xaxis": {
+              "fixedColor": "#C4162A",
-    "show": true,
+              "mode": "fixed"
-    "mode": "time",
+            }
-    "name": null,
+          },
-    "values": [],
+          {
-    "buckets": null
+            "id": "custom.fillOpacity",
+            "value": 0
+          },
+          {
+            "id": "custom.lineWidth",
+            "value": 2
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/trend/"
+        },
+        "properties": [
+          {
+            "id": "custom.fillOpacity",
+            "value": 0
+          },
+          {
+            "id": "custom.lineWidth",
+            "value": 4
+          },
+          {
+            "id": "custom.lineStyle",
+            "value": {
+              "fill": "dash",
+              "dash": [
+                4,
+                10
+              ]
+            }
+          }
+        ]
+      }
+    ]
   },
-  "yaxis": {
+  "maxDataPoints": null,
-    "align": false,
+  "timeFrom": null,
-    "alignLevel": null
+  "timeShift": null
-  },
-  "lines": true,
-  "fill": 1,
-  "linewidth": 1,
-  "dashLength": 10,
-  "spaceLength": 10,
-  "pointradius": 2,
-  "legend": {
-    "show": true,
-    "values": true,
-    "min": false,
-    "max": true,
-    "current": true,
-    "total": false,
-    "avg": true,
-    "alignAsTable": true
-  },
-  "nullPointMode": "connected",
-  "tooltip": {
-    "value_type": "individual",
-    "shared": true,
-    "sort": 0
-  },
-  "aliasColors": {},
-  "seriesOverrides": [
-    {
-      "$$hashKey": "object:486",
-      "alias": "/total/",
-      "fill": 0,
-      "linewidth": 2,
-      "color": "#C4162A",
-      "zindex": 3
-    },
-    {
-      "$$hashKey": "object:829",
-      "alias": "/trend/",
-      "fill": 0,
-      "linewidth": 4,
-      "dashes": true,
-      "dashLength": 4
-    }
-  ],
-  "thresholds": [],
-  "timeRegions": [],
-  "steppedLine": true,
-  "fillGradient": 0,
-  "dashes": false,
-  "hiddenSeries": false,
-  "points": false,
-  "bars": false,
-  "stack": false,
-  "percentage": false,
-  "maxDataPoints": 750,
-  "interval": "30s"
 }

salt/grafana/panels/disk_usage_root_percent_graph.json.jinja

@@ -1,45 +1,82 @@
 {
-  "aliasColors": {},
+  "id": 67830,
-  "dashLength": 10,
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
   "gridPos": {
     "x": {{ PANELS.disk_usage_root_percent_graph.gridPos.x }},
     "y": {{ PANELS.disk_usage_root_percent_graph.gridPos.y }},
     "w": {{ PANELS.disk_usage_root_percent_graph.gridPos.w }},
     "h": {{ PANELS.disk_usage_root_percent_graph.gridPos.h }}
   },
-  "id": 67830,
+  "type": "timeseries",
+  "title": "Disk Usage /",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
   "interval": "30s",
-  "legend": {
+  "fieldConfig": {
-    "alignAsTable": true,
+    "defaults": {
-    "avg": false,
+      "custom": {
-    "current": true,
+        "drawStyle": "line",
-    "max": false,
+        "lineInterpolation": "linear",
-    "min": false,
+        "barAlignment": 0,
-    "rightSide": true,
+        "lineWidth": 1,
-    "show": true,
+        "fillOpacity": 0,
-    "sort": "current",
+        "gradientMode": "none",
-    "sortDesc": true,
+        "spanNulls": false,
-    "total": false,
+        "showPoints": "never",
-    "values": true
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "percent",
+      "decimals": 1,
+      "min": 0,
+      "max": 100
+    },
+    "overrides": []
   },
-  "lines": true,
-  "linewidth": 1,
-  "maxDataPoints": 750,
-  "nullPointMode": "connected",
   "options": {
-    "alertThreshold": false
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "lastNotNull"
+      ]
+    }
   },
-  "pluginVersion": "7.5.4",
-  "pointradius": 2,
-  "renderer": "flot",
-  "seriesOverrides": [],
-  "spaceLength": 10,
-  "steppedLine": true,
   "targets": [
     {
       "alias": "$tag_host $tag_role",
@@ -51,24 +88,25 @@
           "type": "time"
         },
         {
-          "type": "tag",
           "params": [
             "host"
-          ]
+          ],
+          "type": "tag"
         },
         {
-          "type": "tag",
           "params": [
             "role"
-          ]
+          ],
+          "type": "tag"
         },
         {
           "params": [
-            "null"
+            "none"
           ],
           "type": "fill"
         }
       ],
+      "measurement": "disk",
       "orderByTime": "ASC",
       "policy": "default",
       "query": "SELECT mean(total) AS \"total\", mean(used) as \"used\" FROM \"disk\" WHERE \"host\" =~ /$servername$/ AND \"path\" = '/' AND $timeFilter GROUP BY time($__interval), \"host\", \"path\"",
@@ -102,60 +140,10 @@
           "operator": "=",
           "value": "/"
         }
-      ],
+      ]
-      "measurement": "disk"
     }
   ],
-  "thresholds": [],
+  "maxDataPoints": null,
-  "timeRegions": [],
-  "title": "Disk Usage /",
-  "tooltip": {
-    "shared": true,
-    "sort": 2,
-    "value_type": "individual"
-  },
-  "type": "graph",
-  "xaxis": {
-    "buckets": null,
-    "mode": "time",
-    "name": null,
-    "show": true,
-    "values": []
-  },
-  "yaxes": [
-    {
-      "$$hashKey": "object:235",
-      "format": "percent",
-      "label": "",
-      "logBase": 1,
-      "max": "100",
-      "min": "0",
-      "show": true,
-      "decimals": 1
-    },
-    {
-      "$$hashKey": "object:236",
-      "format": "short",
-      "label": null,
-      "logBase": 1,
-      "max": null,
-      "min": null,
-      "show": true
-    }
-  ],
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "fill": 0,
-  "bars": false,
-  "dashes": false,
-  "fillGradient": 0,
-  "hiddenSeries": false,
-  "percentage": false,
-  "points": false,
-  "stack": false,
   "timeFrom": null,
-  "timeShift": null,
+  "timeShift": null
-  "decimals": 1
 }

salt/grafana/panels/elasticsearch_ingest_performance_nontc_graph.json.jinja

@@ -0,0 +1,796 @@
+{
+  "id": 445549,
+  "gridPos": {
+    "x": {{ PANELS.elasticsearch_ingest_performance_nontc_graph.gridPos.x }},
+    "y": {{ PANELS.elasticsearch_ingest_performance_nontc_graph.gridPos.y }},
+    "w": {{ PANELS.elasticsearch_ingest_performance_nontc_graph.gridPos.w }},
+    "h": {{ PANELS.elasticsearch_ingest_performance_nontc_graph.gridPos.h }}
+  },
+  "type": "timeseries",
+  "title": "Elastic Ingest Performance - $searchnode",
+  "repeat": "searchnode",
+  "repeatDirection": "v",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 10,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "decimals": 0,
+      "unit": "ms"
+    },
+    "overrides": []
+  },
+  "options": {
+    "tooltip": {
+      "mode": "multi"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean"
+      ]
+    }
+  },
+  "targets": [
+    {
+      "alias": "community.id_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "queryType": "randomWalk",
+      "refId": "B",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_community_id_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "conditionals_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "queryType": "randomWalk",
+      "refId": "C",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_conditional_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "convert_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "queryType": "randomWalk",
+      "refId": "D",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_convert_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "data.index.name_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "F",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_date_index_name_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "data_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "G",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_date_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "dissect_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "H",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_dissect_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "dot.expander_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "I",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_dot_expander_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "geoip_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "K",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_geoip_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "grok_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "L",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_grok_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "json_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "O",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_json_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "kv_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "P",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_kv_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "lowercase_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "Q",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_lowercase_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "remove_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "R",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_remove_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "rename_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "S",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_rename_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "script_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "T",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_script_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "url_decodes",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "U",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_user_agent_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    }
+  ],
+  "description": "",
+  "timeFrom": null,
+  "timeShift": null
+}

salt/grafana/panels/elasticsearch_ingest_performance_tc_graph.json.jinja

@@ -0,0 +1,793 @@
+{
+  "id": 445548,
+  "gridPos": {
+    "x": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.x }},
+    "y": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.y }},
+    "w": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.w }},
+    "h": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.h }}
+  },
+  "type": "timeseries",
+  "title": "Elastic Ingest Performance",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "options": {
+    "tooltip": {
+      "mode": "multi"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean"
+      ]
+    }
+  },
+  "targets": [
+    {
+      "alias": "community.id_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "queryType": "randomWalk",
+      "refId": "B",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_community_id_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "conditionals_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "queryType": "randomWalk",
+      "refId": "C",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_conditional_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "convert_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "queryType": "randomWalk",
+      "refId": "D",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_convert_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "data.index.name_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "F",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_date_index_name_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "data_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "G",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_date_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "dissect_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "H",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_dissect_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "dot.expander_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "I",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_dot_expander_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "geoip_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "K",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_geoip_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "grok_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "L",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_grok_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "json_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "O",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_json_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "kv_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "P",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_kv_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "lowercase_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "Q",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_lowercase_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "remove_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "R",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_remove_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "rename_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "S",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_rename_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "script_time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "T",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_script_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    },
+    {
+      "alias": "url_decodes",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
+      "queryType": "randomWalk",
+      "rawQuery": false,
+      "refId": "U",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "ingest_processor_stats_user_agent_time_in_millis"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "last"
+          },
+          {
+            "params": [],
+            "type": "non_negative_difference"
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    }
+  ],
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 10,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "ms"
+    },
+    "overrides": []
+  },
+  "description": "",
+  "timeFrom": null,
+  "timeShift": null
+}

salt/grafana/panels/elasticsearch_pipeline_time_nontc_graph.json.jinja

@@ -0,0 +1,153 @@
+{
+  "id": 445552,
+  "gridPos": {
+    "x": {{ PANELS.elasticsearch_pipeline_time_nontc_graph.gridPos.x }},
+    "y": {{ PANELS.elasticsearch_pipeline_time_nontc_graph.gridPos.y }},
+    "w": {{ PANELS.elasticsearch_pipeline_time_nontc_graph.gridPos.w }},
+    "h": {{ PANELS.elasticsearch_pipeline_time_nontc_graph.gridPos.h }}
+  },
+  "type": "timeseries",
+  "title": "Pipeline Time",
+  "datasource": "InfluxDB",
+  "interval": "30s",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 0,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "ms"
+    },
+    "overrides": []
+  },
+  "options": {
+    "tooltip": {
+      "mode": "multi"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean"
+      ]
+    }
+  },
+  "targets": [
+    {
+      "alias": "$tag_host",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "host"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "refId": "A",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_pipeline_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "role",
+          "operator": "=~",
+          "value": "/search/"
+        },
+        {
+          "key": "role",
+          "value": "heavynode",
+          "operator": "=",
+          "condition": "OR"
+        },
+        {
+          "key": "role",
+          "value": "standalone",
+          "operator": "=",
+          "condition": "OR"
+        },
+        {
+          "key": "role",
+          "value": "eval",
+          "operator": "=",
+          "condition": "OR"
+        }
+      ]
+    }
+  ]
+}
+
+
+
+
+
+
+

salt/grafana/panels/elasticsearch_pipeline_time_tc_graph.json.jinja

@@ -0,0 +1,129 @@
+{
+  "id": 445552,
+  "gridPos": {
+    "x": {{ PANELS.elasticsearch_pipeline_time_tc_graph.gridPos.x }},
+    "y": {{ PANELS.elasticsearch_pipeline_time_tc_graph.gridPos.y }},
+    "w": {{ PANELS.elasticsearch_pipeline_time_tc_graph.gridPos.w }},
+    "h": {{ PANELS.elasticsearch_pipeline_time_tc_graph.gridPos.h }}
+  },
+  "type": "timeseries",
+  "title": "Pipeline Time",
+  "datasource": "InfluxDB",
+  "interval": "30s",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 0,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "ms"
+    },
+    "overrides": []
+  },
+  "options": {
+    "tooltip": {
+      "mode": "multi"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean"
+      ]
+    }
+  },
+  "targets": [
+    {
+      "alias": "Time",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "measurement": "elasticsearch_clusterstats_nodes",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "refId": "A",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "ingest_processor_stats_pipeline_time_in_millis"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_difference",
+            "params": []
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "cluster_name",
+          "operator": "=",
+          "value": "$cluster_name"
+        }
+      ]
+    }
+  ]
+}
+
+
+
+
+
+
+

salt/grafana/panels/io_wait_graph.json.jinja

@@ -1,20 +1,131 @@
 {
-  "type": "graph",
+  "id": 69011,
-  "title": "IO Wait",
   "gridPos": {
     "x": {{ PANELS.io_wait_graph.gridPos.x }},
     "y": {{ PANELS.io_wait_graph.gridPos.y }},
     "w": {{ PANELS.io_wait_graph.gridPos.w }},
     "h": {{ PANELS.io_wait_graph.gridPos.h }}
   },
-  "id": 69011,
+  "type": "timeseries",
+  "title": "IO Wait",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 0,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "percent",
+      "min": 0,
+      "decimals": 1
+    },
+    "overrides": []
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
   "targets": [
     {
-      "refId": "A",
+      "alias": "$tag_host $tag_role",
-      "queryType": "randomWalk",
+      "groupBy": [
-      "policy": "default",
+        {
-      "resultFormat": "time_series",
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "host"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "role"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "measurement": "cpu",
       "orderByTime": "ASC",
+      "policy": "default",
+      "queryType": "randomWalk",
+      "refId": "A",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "usage_iowait"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
       "tags": [
         {
           "key": "host",
@@ -27,129 +138,11 @@
           "operator": "=",
           "value": "cpu-total"
         }
-      ],
+      ]
-      "groupBy": [
-        {
-          "type": "time",
-          "params": [
-            "$__interval"
-          ]
-        },
-        {
-          "type": "tag",
-          "params": [
-            "host"
-          ]
-        },
-        {
-          "type": "tag",
-          "params": [
-            "role"
-          ]
-        },
-        {
-          "type": "fill",
-          "params": [
-            "null"
-          ]
-        }
-      ],
-      "select": [
-        [
-          {
-            "type": "field",
-            "params": [
-              "usage_iowait"
-            ]
-          },
-          {
-            "type": "mean",
-            "params": []
-          }
-        ]
-      ],
-      "measurement": "cpu",
-      "alias": "$tag_host $tag_role"
     }
   ],
-  "options": {
+  "description": "",
-    "alertThreshold": true
+  "timeFrom": null,
-  },
+  "timeShift": null,
-  "datasource": "InfluxDB",
+  "interval": "30s"
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
-  "pluginVersion": "7.5.4",
-  "renderer": "flot",
-  "yaxes": [
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": "0",
-      "max": null,
-      "format": "percent",
-      "$$hashKey": "object:1740"
-    },
-    {
-      "label": null,
-      "show": true,
-      "logBase": 1,
-      "min": null,
-      "max": null,
-      "format": "short",
-      "$$hashKey": "object:1741"
-    }
-  ],
-  "xaxis": {
-    "show": true,
-    "mode": "time",
-    "name": null,
-    "values": [],
-    "buckets": null
-  },
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "lines": true,
-  "linewidth": 1,
-  "dashLength": 10,
-  "spaceLength": 10,
-  "pointradius": 2,
-  "legend": {
-    "alignAsTable": true,
-    "avg": true,
-    "current": true,
-    "max": true,
-    "min": false,
-    "rightSide": true,
-    "show": true,
-    "sort": "current",
-    "sortDesc": true,
-    "total": false,
-    "values": true
-  },
-  "nullPointMode": "connected",
-  "tooltip": {
-    "value_type": "individual",
-    "shared": true,
-    "sort": 2
-  },
-  "aliasColors": {},
-  "seriesOverrides": [],
-  "thresholds": [],
-  "timeRegions": [],
-  "fill": 0,
-  "fillGradient": 0,
-  "dashes": false,
-  "hiddenSeries": false,
-  "points": false,
-  "bars": false,
-  "stack": false,
-  "percentage": false,
-  "steppedLine": false,
-  "decimals": 1,
-  "description": ""
 }

salt/grafana/panels/io_wait_stat.json.jinja

@@ -1,5 +1,16 @@
 {
+  "id": 61867,
+  "gridPos": {
+    "x": {{ PANELS.io_wait_stat.gridPos.x }},
+    "y": {{ PANELS.io_wait_stat.gridPos.y }},
+    "w": {{ PANELS.io_wait_stat.gridPos.w }},
+    "h": {{ PANELS.io_wait_stat.gridPos.h }}
+  },
+  "type": "stat",
+  "title": "IOWait",
   "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "links": [],
   "fieldConfig": {
     "defaults": {
       "thresholds": {
@@ -21,31 +32,41 @@
       },
       "mappings": [
         {
-          "op": "=",
+          "options": {
-          "text": "N/A",
+            "match": "null",
-          "value": "null",
+            "result": {
-          "$$hashKey": "object:1217",
+              "text": "N/A"
-          "id": 0,
+            }
-          "type": 1
+          },
+          "type": "special"
         }
       ],
-      "unit": "percent",
-      "decimals": 2,
       "color": {
         "mode": "thresholds"
-      }
+      },
+      "decimals": 2,
+      "max": 100,
+      "min": 0,
+      "unit": "percent"
     },
     "overrides": []
   },
-  "gridPos": {
+  "interval": "30",
-    "x": {{ PANELS.io_wait_stat.gridPos.x }},
+  "options": {
-    "y": {{ PANELS.io_wait_stat.gridPos.y }},
+    "reduceOptions": {
-    "w": {{ PANELS.io_wait_stat.gridPos.w }},
+      "values": false,
-    "h": {{ PANELS.io_wait_stat.gridPos.h }}
+      "calcs": [
+        "lastNotNull"
+      ],
+      "fields": ""
+    },
+    "orientation": "horizontal",
+    "text": {},
+    "textMode": "auto",
+    "colorMode": "value",
+    "graphMode": "area",
+    "justifyMode": "auto"
   },
-  "id": 61867,
-  "links": [],
-  "maxDataPoints": 100,
   "targets": [
     {
       "dsType": "influxdb",
@@ -64,6 +85,7 @@
         }
       ],
       "measurement": "cpu",
+      "orderByTime": "ASC",
       "policy": "default",
       "query": "SELECT non_negative_derivative(mean(\"usage_iowait\"), 1s) FROM \"cpu\" WHERE (host =~ /$servername$/ AND \"cpu\" = 'cpu-total') AND $timeFilter GROUP BY time($interval) fill(null)",
       "rawQuery": false,
@@ -86,8 +108,8 @@
       "tags": [
         {
           "key": "host",
-          "operator": "=~",
+          "operator": "=",
-          "value": "/^$servername$/"
+          "value": "$servername"
         },
         {
           "condition": "AND",
@@ -95,28 +117,9 @@
           "operator": "=",
           "value": "cpu-total"
         }
-      ],
+      ]
-      "orderByTime": "ASC"
     }
   ],
-  "title": "IOWait",
+  "maxDataPoints": null,
-  "type": "stat",
+  "cacheTimeout": null
-  "options": {
-    "reduceOptions": {
-      "values": false,
-      "calcs": [
-        "lastNotNull"
-      ],
-      "fields": ""
-    },
-    "orientation": "horizontal",
-    "text": {},
-    "textMode": "auto",
-    "colorMode": "value",
-    "graphMode": "area",
-    "justifyMode": "auto"
-  },
-  "cacheTimeout": null,
-  "interval": null,
-  "pluginVersion": "7.5.4"
 }

salt/grafana/panels/load_average_5_minute_stat.json.jinja

@@ -1,5 +1,17 @@
 {
+  "id": 61859,
+  "gridPos": {
+    "x": {{ PANELS.load_average_5_minute_stat.gridPos.x }},
+    "y": {{ PANELS.load_average_5_minute_stat.gridPos.y }},
+    "w": {{ PANELS.load_average_5_minute_stat.gridPos.w }},
+    "h": {{ PANELS.load_average_5_minute_stat.gridPos.h }}
+  },
+  "type": "stat",
+  "title": "5 Minute Load Average - $cpucount Cores",
   "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30",
+  "links": [],
   "fieldConfig": {
     "defaults": {
       "thresholds": {
@@ -7,7 +19,7 @@
         "steps": [
           {
             "color": "rgba(50, 172, 45, 0.97)",
-            "value": "$cpucount / 2"
+            "value": null
           },
           {
             "color": "rgba(237, 129, 40, 0.89)",
@@ -21,30 +33,39 @@
       },
       "mappings": [
         {
-          "op": "=",
+          "options": {
-          "text": "N/A",
+            "from": null,
-          "value": "null",
+            "result": {
-          "id": 0,
+              "text": "N/A"
-          "type": 2
+            },
+            "to": null
+          },
+          "type": "range"
         }
       ],
-      "unit": "none",
-      "decimals": 1,
       "color": {
         "mode": "thresholds"
-      }
+      },
+      "decimals": 1,
+      "unit": "none"
     },
     "overrides": []
   },
-  "gridPos": {
+  "options": {
-    "x": {{ PANELS.load_average_5_minute_stat.gridPos.x }},
+    "reduceOptions": {
-    "y": {{ PANELS.load_average_5_minute_stat.gridPos.y }},
+      "values": false,
-    "w": {{ PANELS.load_average_5_minute_stat.gridPos.w }},
+      "calcs": [
-    "h": {{ PANELS.load_average_5_minute_stat.gridPos.h }}
+        "lastNotNull"
+      ],
+      "fields": ""
+    },
+    "orientation": "horizontal",
+    "text": {},
+    "textMode": "auto",
+    "colorMode": "value",
+    "graphMode": "area",
+    "justifyMode": "auto"
   },
-  "id": 61859,
-  "links": [],
-  "maxDataPoints": 100,
   "targets": [
     {
       "dsType": "influxdb",
@@ -65,6 +86,8 @@
       "measurement": "system",
       "orderByTime": "ASC",
       "policy": "default",
+      "query": "SELECT last(\"load5\") FROM \"system\" WHERE (\"host\" = '$servername') AND $timeFilter GROUP BY time($__interval) fill(null)",
+      "rawQuery": false,
       "refId": "A",
       "resultFormat": "time_series",
       "select": [
@@ -84,30 +107,12 @@
       "tags": [
         {
           "key": "host",
-          "operator": "=~",
+          "operator": "=",
-          "value": "/^$servername$/"
+          "value": "$servername"
         }
       ]
     }
   ],
-  "title": "5 Minute Load Average - $cpucount Cores",
-  "type": "stat",
-  "options": {
-    "reduceOptions": {
-      "values": false,
-      "calcs": [
-        "lastNotNull"
-      ],
-      "fields": ""
-    },
-    "orientation": "horizontal",
-    "text": {},
-    "textMode": "auto",
-    "colorMode": "value",
-    "graphMode": "area",
-    "justifyMode": "auto"
-  },
-  "pluginVersion": "7.5.4",
   "cacheTimeout": null,
-  "interval": null
+  "maxDataPoints": null
 }

salt/grafana/panels/load_averages_graph.json.jinja

@@ -1,48 +1,30 @@
 {
-  "aliasColors": {},
+  "id": 61869,
-  "dashLength": 10,
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
   "gridPos": {
     "x": {{ PANELS.load_averages_graph.gridPos.x }},
     "y": {{ PANELS.load_averages_graph.gridPos.y }},
     "w": {{ PANELS.load_averages_graph.gridPos.w }},
     "h": {{ PANELS.load_averages_graph.gridPos.h }}
   },
-  "id": 61869,
+  "type": "timeseries",
-  "legend": {
+  "title": "1 Minute Load Average",
-    "alignAsTable": true,
+  "datasource": "InfluxDB",
-    "avg": true,
+  "pluginVersion": "8.2.1",
-    "current": true,
+  "interval": "30s",
-    "max": true,
-    "min": true,
-    "show": true,
-    "total": false,
-    "values": true
-  },
-  "lines": true,
-  "linewidth": 1,
-  "nullPointMode": "connected",
   "options": {
-    "alertThreshold": true
+    "tooltip": {
-  },
+      "mode": "single"
-  "pluginVersion": "7.5.4",
+    },
-  "pointradius": 2,
+    "legend": {
-  "renderer": "flot",
+      "displayMode": "table",
-  "seriesOverrides": [
+      "placement": "bottom",
-    {
+      "calcs": [
-      "$$hashKey": "object:364",
+        "max",
-      "alias": "/trend/",
+        "mean",
-      "fill": 0,
+        "lastNotNull"
-      "linewidth": 4,
+      ]
-      "dashes": true,
-      "dashLength": 4
     }
-  ],
+  },
-  "spaceLength": 10,
   "targets": [
     {
       "alias": "$tag_host: $col",
@@ -62,7 +44,7 @@
       ],
       "orderByTime": "ASC",
       "policy": "default",
-      "query": "SELECT mean(load1) as \"1 minute\", mean(load5) as \"5 minutes\", mean(load15) as \"15 minutes\" FROM \"system\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), * ORDER BY asc",
+      "query": "SELECT mean(load1) as \"1 minute\", last(n_cpus) as \"Total Cores\" FROM \"system\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), * ORDER BY asc",
       "queryType": "randomWalk",
       "rawQuery": true,
       "refId": "A",
@@ -99,9 +81,10 @@
           "type": "fill"
         }
       ],
+      "hide": false,
       "orderByTime": "ASC",
       "policy": "default",
-      "query": "SELECT mean(mean_load1) as \"trend_1 minute\", mean(mean_load5) as \"trend_5 minutes\", mean(mean_load15) as \"trend_15 minutes\" FROM \"so_long_term\".\"system\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), * ORDER BY asc",
+      "query": "SELECT mean(mean_load1) as \"trend_1 minute\" FROM \"so_long_term\".\"system\" WHERE host =~ /$servername$/ AND $timeFilter GROUP BY time($__interval), * fill(linear) ORDER BY asc",
       "queryType": "randomWalk",
       "rawQuery": true,
       "refId": "B",
@@ -120,61 +103,85 @@
           }
         ]
       ],
-      "tags": [],
+      "tags": []
-      "hide": false
     }
   ],
-  "thresholds": [],
+  "fieldConfig": {
-  "timeRegions": [],
+    "defaults": {
-  "title": "Load Averages - $cpucount Cores",
+      "custom": {
-  "tooltip": {
+        "drawStyle": "line",
-    "shared": true,
+        "lineInterpolation": "linear",
-    "sort": 0,
+        "barAlignment": 0,
-    "value_type": "individual"
+        "lineWidth": 1,
-  },
+        "fillOpacity": 0,
-  "type": "graph",
+        "gradientMode": "none",
-  "xaxis": {
+        "spanNulls": false,
-    "buckets": null,
+        "showPoints": "never",
-    "mode": "time",
+        "pointSize": 5,
-    "name": null,
+        "stacking": {
-    "show": true,
+          "mode": "none",
-    "values": []
+          "group": "A"
-  },
+        },
-  "yaxes": [
+        "axisPlacement": "auto",
-    {
+        "axisLabel": "",
-      "$$hashKey": "object:287",
+        "scaleDistribution": {
-      "format": "short",
+          "type": "linear"
-      "label": null,
+        },
-      "logBase": 1,
+        "hideFrom": {
-      "max": null,
+          "tooltip": false,
-      "min": null,
+          "viz": false,
-      "show": true
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "short",
+      "decimals": 1
     },
-    {
+    "overrides": [
-      "$$hashKey": "object:288",
+      {
-      "format": "short",
+        "matcher": {
-      "label": null,
+          "id": "byRegexp",
-      "logBase": 1,
+          "options": "/trend/"
-      "max": null,
+        },
-      "min": null,
+        "properties": [
-      "show": true
+          {
-    }
+            "id": "custom.fillOpacity",
-  ],
+            "value": 0
-  "yaxis": {
+          },
-    "align": false,
+          {
-    "alignLevel": null
+            "id": "custom.lineWidth",
+            "value": 4
+          },
+          {
+            "id": "custom.lineStyle",
+            "value": {
+              "fill": "dash",
+              "dash": [
+                4,
+                10
+              ]
+            }
+          }
+        ]
+      }
+    ]
   },
-  "bars": false,
+  "maxDataPoints": null,
-  "dashes": false,
-  "fill": 0,
-  "fillGradient": 0,
-  "hiddenSeries": false,
-  "percentage": false,
-  "points": false,
-  "stack": false,
-  "steppedLine": false,
   "timeFrom": null,
-  "timeShift": null,
+  "timeShift": null
-  "maxDataPoints": 750,
-  "interval": "30s"
 }

salt/grafana/panels/logstash_eps_in_out_manager_graph.json.jinja

@@ -0,0 +1,403 @@
+{
+  "id": 77741,
+  "gridPos": {
+    "x": {{ PANELS.logstash_eps_in_out_manager_graph.gridPos.x }},
+    "y": {{ PANELS.logstash_eps_in_out_manager_graph.gridPos.y }},
+    "w": {{ PANELS.logstash_eps_in_out_manager_graph.gridPos.w }},
+    "h": {{ PANELS.logstash_eps_in_out_manager_graph.gridPos.h }}
+  },
+  "type": "timeseries",
+  "title": "Manager Logstash Events",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "description": "Events from the grid to redis",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 50,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "EPS",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "decimals": 2,
+      "unit": "short"
+    },
+    "overrides": [
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/Incoming/"
+        },
+        "properties": [
+          {
+            "id": "color",
+            "value": {
+              "fixedColor": "orange",
+              "mode": "fixed"
+            }
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/Outgoing/"
+        },
+        "properties": [
+          {
+            "id": "color",
+            "value": {
+              "fixedColor": "green",
+              "mode": "fixed"
+            }
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byName",
+          "options": "Incoming hidden"
+        },
+        "properties": [
+          {
+            "id": "custom.fillBelowTo",
+            "value": "Outgoing hidden"
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byName",
+          "options": "Outgoing hidden"
+        },
+        "properties": [
+          {
+            "id": "custom.fillBelowTo",
+            "value": "Incoming hidden"
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byName",
+          "options": "Incoming"
+        },
+        "properties": [
+          {
+            "id": "custom.fillOpacity",
+            "value": 0
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byName",
+          "options": "Outgoing"
+        },
+        "properties": [
+          {
+            "id": "custom.fillOpacity",
+            "value": 0
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/hidden/"
+        },
+        "properties": [
+          {
+            "id": "custom.hideFrom",
+            "value": {
+              "legend": true,
+              "tooltip": true,
+              "viz": false
+            }
+          }
+        ]
+      }
+    ]
+  },
+  "options": {
+    "tooltip": {
+      "mode": "multi"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "bottom",
+      "calcs": [
+        "max",
+        "mean"
+      ]
+    }
+  },
+  "targets": [
+    {
+      "alias": "Incoming",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "logstash_events",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "queryType": "randomWalk",
+      "refId": "A",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "in"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "role",
+          "operator": "=~",
+          "value": "/^manager/"
+        },
+        {
+          "key": "role",
+          "value": "standalone",
+          "operator": "=",
+          "condition": "OR"
+        },
+        {
+          "key": "role",
+          "value": "eval",
+          "operator": "=",
+          "condition": "OR"
+        }
+      ]
+    },
+    {
+      "alias": "Outgoing",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "logstash_events",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "queryType": "randomWalk",
+      "refId": "B",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "out"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "role",
+          "operator": "=~",
+          "value": "/^manager/"
+        },
+        {
+          "key": "role",
+          "value": "standalone",
+          "operator": "=",
+          "condition": "OR"
+        },
+        {
+          "key": "role",
+          "value": "eval",
+          "operator": "=",
+          "condition": "OR"
+        }
+      ]
+    },
+    {
+      "alias": "Incoming hidden",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "logstash_events",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "queryType": "randomWalk",
+      "refId": "C",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "in"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "role",
+          "operator": "=~",
+          "value": "/^manager/"
+        },
+        {
+          "key": "role",
+          "value": "standalone",
+          "operator": "=",
+          "condition": "OR"
+        },
+        {
+          "key": "role",
+          "value": "eval",
+          "operator": "=",
+          "condition": "OR"
+        }
+      ]
+    },
+    {
+      "alias": "Outgoing hidden",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        }
+      ],
+      "hide": false,
+      "measurement": "logstash_events",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "queryType": "randomWalk",
+      "refId": "D",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "out"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "role",
+          "operator": "=~",
+          "value": "/^manager/"
+        },
+        {
+          "key": "role",
+          "value": "standalone",
+          "operator": "=",
+          "condition": "OR"
+        },
+        {
+          "key": "role",
+          "value": "eval",
+          "operator": "=",
+          "condition": "OR"
+        }
+      ]
+    }
+  ],
+  "timeFrom": null,
+  "timeShift": null
+}

salt/grafana/panels/logstash_estimated_eps_graph.json.jinja

@@ -1,192 +0,0 @@
-{
-  "aliasColors": {},
-  "bars": false,
-  "maxDataPoints": 750,
-  "interval": "30s",
-  "dashLength": 10,
-  "dashes": false,
-  "datasource": "InfluxDB",
-  "description": "",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
-  "fill": 1,
-  "fillGradient": 0,
-  "gridPos": {
-    "x": {{ PANELS.logstash_estimated_eps_graph.gridPos.x }},
-    "y": {{ PANELS.logstash_estimated_eps_graph.gridPos.y }},
-    "w": {{ PANELS.logstash_estimated_eps_graph.gridPos.w }},
-    "h": {{ PANELS.logstash_estimated_eps_graph.gridPos.h }}
-  },
-  "hiddenSeries": false,
-  "id": 76,
-  "legend": {
-    "alignAsTable": true,
-    "avg": true,
-    "current": true,
-    "hideEmpty": true,
-    "max": true,
-    "min": false,
-    "rightSide": false,
-    "show": true,
-    "sort": "current",
-    "sortDesc": true,
-    "total": false,
-    "values": true
-  },
-  "lines": true,
-  "linewidth": 1,
-  "nullPointMode": "connected",
-  "options": {
-    "alertThreshold": false
-  },
-  "percentage": false,
-  "pluginVersion": "7.5.4",
-  "pointradius": 2,
-  "points": false,
-  "renderer": "flot",
-  "seriesOverrides": [
-    {
-      "alias": "/Trend/",
-      "dashLength": 4,
-      "dashes": true,
-      "fill": 0,
-      "linewidth": 4
-    }
-  ],
-  "spaceLength": 10,
-  "stack": false,
-  "steppedLine": false,
-  "targets": [
-    {
-      "alias": "EPS Current",
-      "groupBy": [
-        {
-          "params": [
-            "$__interval"
-          ],
-          "type": "time"
-        },
-        {
-          "params": [
-            "null"
-          ],
-          "type": "fill"
-        }
-      ],
-      "measurement": "consumptioneps",
-      "orderByTime": "ASC",
-      "policy": "default",
-      "queryType": "randomWalk",
-      "refId": "A",
-      "resultFormat": "time_series",
-      "select": [
-        [
-          {
-            "params": [
-              "eps"
-            ],
-            "type": "field"
-          },
-          {
-            "params": [],
-            "type": "mean"
-          }
-        ]
-      ],
-      "tags": [
-        {
-          "key": "host",
-          "operator": "=",
-          "value": "$servername"
-        }
-      ]
-    },
-    {
-      "alias": "EPS Trend",
-      "groupBy": [
-        {
-          "params": [
-            "$__interval"
-          ],
-          "type": "time"
-        },
-        {
-          "params": [
-            "null"
-          ],
-          "type": "fill"
-        }
-      ],
-      "hide": false,
-      "measurement": "consumptioneps",
-      "orderByTime": "ASC",
-      "policy": "so_long_term",
-      "queryType": "randomWalk",
-      "refId": "B",
-      "resultFormat": "time_series",
-      "select": [
-        [
-          {
-            "params": [
-              "mean_eps"
-            ],
-            "type": "field"
-          },
-          {
-            "params": [],
-            "type": "mean"
-          }
-        ]
-      ],
-      "tags": [
-        {
-          "key": "host",
-          "operator": "=",
-          "value": "$servername"
-        }
-      ]
-    }
-  ],
-  "thresholds": [],
-  "timeFrom": null,
-  "timeRegions": [],
-  "timeShift": null,
-  "title": "Estimated EPS",
-  "tooltip": {
-    "shared": true,
-    "sort": 0,
-    "value_type": "individual"
-  },
-  "type": "graph",
-  "xaxis": {
-    "buckets": null,
-    "mode": "time",
-    "name": null,
-    "show": true,
-    "values": []
-  },
-  "yaxes": [
-    {
-      "format": "short",
-      "label": "EPS",
-      "logBase": 1,
-      "max": null,
-      "min": null,
-      "show": true
-    },
-    {
-      "format": "short",
-      "label": null,
-      "logBase": 1,
-      "max": null,
-      "min": null,
-      "show": false
-    }
-  ],
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  }
-}

salt/grafana/panels/logstash_estimated_eps_in_graph.json.jinja

@@ -0,0 +1,230 @@
+{
+  "id": 76,
+  "gridPos": {
+    "x": {{ PANELS.logstash_estimated_eps_in_graph.gridPos.x }},
+    "y": {{ PANELS.logstash_estimated_eps_in_graph.gridPos.y }},
+    "w": {{ PANELS.logstash_estimated_eps_in_graph.gridPos.w }},
+    "h": {{ PANELS.logstash_estimated_eps_in_graph.gridPos.h }}
+  },
+  "type": "timeseries",
+  "title": "Estimated EPS In",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 10,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "EPS",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "short",
+      "decimals": 1
+    },
+    "overrides": [
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/trend/"
+        },
+        "properties": [
+          {
+            "id": "custom.fillOpacity",
+            "value": 0
+          },
+          {
+            "id": "custom.lineWidth",
+            "value": 4
+          },
+          {
+            "id": "custom.lineStyle",
+            "value": {
+              "fill": "dash",
+              "dash": [
+                4,
+                10
+              ]
+            }
+          }
+        ]
+      }
+    ]
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
+  "targets": [
+    {
+      "refId": "A",
+      "hide": false,
+      "policy": "default",
+      "resultFormat": "time_series",
+      "orderByTime": "ASC",
+      "tags": [
+        {
+          "key": "host",
+          "value": "/^$servername$/",
+          "operator": "=~"
+        }
+      ],
+      "groupBy": [
+        {
+          "type": "time",
+          "params": [
+            "$__interval"
+          ]
+        },
+        {
+          "type": "tag",
+          "params": [
+            "host"
+          ]
+        },
+        {
+          "type": "fill",
+          "params": [
+            "null"
+          ]
+        }
+      ],
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "in"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "measurement": "logstash_events",
+      "alias": "$tag_host: $col",
+      "query": "SELECT non_negative_derivative(mean(\"in\"), 1s) as \"current_in\" FROM \"logstash_events\" WHERE (\"host\" =~ /^$servername$/) AND $timeFilter GROUP BY time($__interval), \"host\" fill(null)",
+      "rawQuery": true
+    },
+    {
+      "refId": "B",
+      "hide": false,
+      "policy": "so_long_term",
+      "resultFormat": "time_series",
+      "orderByTime": "ASC",
+      "tags": [
+        {
+          "key": "host",
+          "value": "/^$servername$/",
+          "operator": "=~"
+        }
+      ],
+      "groupBy": [
+        {
+          "type": "time",
+          "params": [
+            "$__interval"
+          ]
+        },
+        {
+          "type": "tag",
+          "params": [
+            "host"
+          ]
+        },
+        {
+          "type": "fill",
+          "params": [
+            "null"
+          ]
+        }
+      ],
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "mean_in"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "measurement": "logstash_events",
+      "alias": "$tag_host: $col",
+      "query": "SELECT non_negative_derivative(mean(\"mean_in\"), 1s) as \"trend_in\" FROM \"so_long_term\".\"logstash_events\" WHERE (\"host\" =~ /^$servername$/) AND $timeFilter GROUP BY time($__interval), \"host\" fill(null)",
+      "rawQuery": true
+    }
+  ],
+  "maxDataPoints": null,
+  "description": "",
+  "timeFrom": null,
+  "timeShift": null,
+  "transformations": []
+}

salt/grafana/panels/logstash_estimated_eps_in_stat.json.jinja

@@ -0,0 +1,136 @@
+{
+  "id": 23,
+  "gridPos": {
+    "x": {{ PANELS.logstash_estimated_eps_in_stat.gridPos.x }},
+    "y": {{ PANELS.logstash_estimated_eps_in_stat.gridPos.y }},
+    "w": {{ PANELS.logstash_estimated_eps_in_stat.gridPos.w }},
+    "h": {{ PANELS.logstash_estimated_eps_in_stat.gridPos.h }}
+  },
+  "type": "stat",
+  "title": "Estimated EPS In - Selected Total",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "links": [],
+  "fieldConfig": {
+    "defaults": {
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "dark-red",
+            "value": null
+          },
+          {
+            "color": "dark-green",
+            "value": 1
+          }
+        ]
+      },
+      "mappings": [
+        {
+          "type": "special",
+          "options": {
+            "match": "null",
+            "result": {
+              "text": "N/A"
+            }
+          }
+        }
+      ],
+      "color": {
+        "mode": "thresholds"
+      },
+      "decimals": 0,
+      "unit": "short"
+    },
+    "overrides": []
+  },
+  "options": {
+    "reduceOptions": {
+      "values": false,
+      "calcs": [
+        "lastNotNull"
+      ],
+      "fields": ""
+    },
+    "orientation": "horizontal",
+    "text": {},
+    "textMode": "value",
+    "colorMode": "value",
+    "graphMode": "area",
+    "justifyMode": "auto"
+  },
+  "targets": [
+    {
+      "refId": "A",
+      "hide": false,
+      "policy": "default",
+      "resultFormat": "time_series",
+      "orderByTime": "ASC",
+      "tags": [
+        {
+          "key": "host",
+          "value": "/^$servername$/",
+          "operator": "=~"
+        }
+      ],
+      "groupBy": [
+        {
+          "type": "time",
+          "params": [
+            "$__interval"
+          ]
+        },
+        {
+          "type": "tag",
+          "params": [
+            "host"
+          ]
+        },
+        {
+          "type": "fill",
+          "params": [
+            "null"
+          ]
+        }
+      ],
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "in"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "measurement": "logstash_events"
+    }
+  ],
+  "transformations": [
+    {
+      "id": "calculateField",
+      "options": {
+        "mode": "reduceRow",
+        "reduce": {
+          "reducer": "sum"
+        },
+        "replaceFields": true
+      }
+    }
+  ],
+  "maxDataPoints": null,
+  "cacheTimeout": null,
+  "timeFrom": null
+}

salt/grafana/panels/logstash_estimated_eps_in_total_graph.json.jinja

@@ -0,0 +1,156 @@
+{
+  "id": 69001,
+  "gridPos": {
+    "x": {{ PANELS.logstash_estimated_eps_in_total_graph.gridPos.x }},
+    "y": {{ PANELS.logstash_estimated_eps_in_total_graph.gridPos.y }},
+    "w": {{ PANELS.logstash_estimated_eps_in_total_graph.gridPos.w }},
+    "h": {{ PANELS.logstash_estimated_eps_in_total_graph.gridPos.h }}
+  },
+  "type": "timeseries",
+  "title": "Estimated EPS In - Selected Total",
+  "transformations": [
+    {
+      "id": "calculateField",
+      "options": {
+        "mode": "reduceRow",
+        "reduce": {
+          "reducer": "sum"
+        },
+        "replaceFields": true,
+        "alias": "Total EPS"
+      }
+    }
+  ],
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 10,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "EPS",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "short",
+      "decimals": 1
+    },
+    "overrides": []
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
+  "targets": [
+    {
+      "refId": "A",
+      "hide": false,
+      "policy": "default",
+      "resultFormat": "time_series",
+      "orderByTime": "ASC",
+      "tags": [
+        {
+          "key": "host",
+          "value": "/^$servername$/",
+          "operator": "=~"
+        }
+      ],
+      "groupBy": [
+        {
+          "type": "time",
+          "params": [
+            "$__interval"
+          ]
+        },
+        {
+          "type": "tag",
+          "params": [
+            "host"
+          ]
+        },
+        {
+          "type": "fill",
+          "params": [
+            "null"
+          ]
+        }
+      ],
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "in"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "measurement": "logstash_events",
+      "query": "SELECT non_negative_derivative(mean(\"in\"), 1s) FROM \"logstash_events\" WHERE (\"host\" =~ /^$servername$/) AND $timeFilter GROUP BY time($__interval), \"host\" fill(null)",
+      "rawQuery": false
+    }
+  ],
+  "maxDataPoints": null,
+  "description": "",
+  "timeFrom": null,
+  "timeShift": null
+}

salt/grafana/panels/logstash_estimated_eps_out_graph.json.jinja

@@ -0,0 +1,230 @@
+{
+  "id": 69000,
+  "gridPos": {
+    "x": {{ PANELS.logstash_estimated_eps_out_graph.gridPos.x }},
+    "y": {{ PANELS.logstash_estimated_eps_out_graph.gridPos.y }},
+    "w": {{ PANELS.logstash_estimated_eps_out_graph.gridPos.w }},
+    "h": {{ PANELS.logstash_estimated_eps_out_graph.gridPos.h }}
+  },
+  "type": "timeseries",
+  "title": "Estimated EPS Out",
+  "transformations": [],
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 10,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "EPS",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "short",
+      "decimals": 1
+    },
+    "overrides": [
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/trend/"
+        },
+        "properties": [
+          {
+            "id": "custom.fillOpacity",
+            "value": 0
+          },
+          {
+            "id": "custom.lineWidth",
+            "value": 4
+          },
+          {
+            "id": "custom.lineStyle",
+            "value": {
+              "fill": "dash",
+              "dash": [
+                4,
+                10
+              ]
+            }
+          }
+        ]
+      }
+    ]
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
+  "targets": [
+    {
+      "refId": "A",
+      "hide": false,
+      "policy": "default",
+      "resultFormat": "time_series",
+      "orderByTime": "ASC",
+      "tags": [
+        {
+          "key": "host",
+          "value": "/^$servername$/",
+          "operator": "=~"
+        }
+      ],
+      "groupBy": [
+        {
+          "type": "time",
+          "params": [
+            "$__interval"
+          ]
+        },
+        {
+          "type": "tag",
+          "params": [
+            "host"
+          ]
+        },
+        {
+          "type": "fill",
+          "params": [
+            "null"
+          ]
+        }
+      ],
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "in"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "measurement": "logstash_events",
+      "alias": "$tag_host: $col",
+      "query": "SELECT non_negative_derivative(mean(\"out\"), 1s) as \"current_out\" FROM \"logstash_events\" WHERE (\"host\" =~ /^$servername$/) AND $timeFilter GROUP BY time($__interval), \"host\" fill(null)",
+      "rawQuery": true
+    },
+    {
+      "refId": "B",
+      "hide": false,
+      "policy": "so_long_term",
+      "resultFormat": "time_series",
+      "orderByTime": "ASC",
+      "tags": [
+        {
+          "key": "host",
+          "value": "/^$servername$/",
+          "operator": "=~"
+        }
+      ],
+      "groupBy": [
+        {
+          "type": "time",
+          "params": [
+            "$__interval"
+          ]
+        },
+        {
+          "type": "tag",
+          "params": [
+            "host"
+          ]
+        },
+        {
+          "type": "fill",
+          "params": [
+            "null"
+          ]
+        }
+      ],
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "mean_in"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "measurement": "logstash_events",
+      "alias": "$tag_host: $col",
+      "query": "SELECT non_negative_derivative(mean(\"mean_out\"), 1s) as \"trend_out\" FROM \"so_long_term\".\"logstash_events\" WHERE (\"host\" =~ /^$servername$/) AND $timeFilter GROUP BY time($__interval), \"host\" fill(null)",
+      "rawQuery": true
+    }
+  ],
+  "maxDataPoints": null,
+  "description": "",
+  "timeFrom": null,
+  "timeShift": null
+}

salt/grafana/panels/logstash_estimated_eps_out_stat.json.jinja

@@ -0,0 +1,136 @@
+{
+  "id": 22323,
+  "gridPos": {
+    "x": {{ PANELS.logstash_estimated_eps_out_stat.gridPos.x }},
+    "y": {{ PANELS.logstash_estimated_eps_out_stat.gridPos.y }},
+    "w": {{ PANELS.logstash_estimated_eps_out_stat.gridPos.w }},
+    "h": {{ PANELS.logstash_estimated_eps_out_stat.gridPos.h }}
+  },
+  "type": "stat",
+  "title": "Estimated EPS Out - Selected Total",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "links": [],
+  "fieldConfig": {
+    "defaults": {
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "dark-red",
+            "value": null
+          },
+          {
+            "color": "dark-green",
+            "value": 1
+          }
+        ]
+      },
+      "mappings": [
+        {
+          "type": "special",
+          "options": {
+            "match": "null",
+            "result": {
+              "text": "N/A"
+            }
+          }
+        }
+      ],
+      "color": {
+        "mode": "thresholds"
+      },
+      "decimals": 0,
+      "unit": "short"
+    },
+    "overrides": []
+  },
+  "options": {
+    "reduceOptions": {
+      "values": false,
+      "calcs": [
+        "lastNotNull"
+      ],
+      "fields": ""
+    },
+    "orientation": "horizontal",
+    "text": {},
+    "textMode": "value",
+    "colorMode": "value",
+    "graphMode": "area",
+    "justifyMode": "auto"
+  },
+  "targets": [
+    {
+      "refId": "A",
+      "hide": false,
+      "policy": "default",
+      "resultFormat": "time_series",
+      "orderByTime": "ASC",
+      "tags": [
+        {
+          "key": "host",
+          "value": "/^$servername$/",
+          "operator": "=~"
+        }
+      ],
+      "groupBy": [
+        {
+          "type": "time",
+          "params": [
+            "$__interval"
+          ]
+        },
+        {
+          "type": "tag",
+          "params": [
+            "host"
+          ]
+        },
+        {
+          "type": "fill",
+          "params": [
+            "null"
+          ]
+        }
+      ],
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "out"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "measurement": "logstash_events"
+    }
+  ],
+  "transformations": [
+    {
+      "id": "calculateField",
+      "options": {
+        "mode": "reduceRow",
+        "reduce": {
+          "reducer": "sum"
+        },
+        "replaceFields": true
+      }
+    }
+  ],
+  "maxDataPoints": null,
+  "cacheTimeout": null,
+  "timeFrom": null
+}

salt/grafana/panels/logstash_estimated_eps_out_total_graph.json.jinja

@@ -0,0 +1,156 @@
+{
+  "id": 69002,
+  "gridPos": {
+    "x": {{ PANELS.logstash_estimated_eps_out_total_graph.gridPos.x }},
+    "y": {{ PANELS.logstash_estimated_eps_out_total_graph.gridPos.y }},
+    "w": {{ PANELS.logstash_estimated_eps_out_total_graph.gridPos.w }},
+    "h": {{ PANELS.logstash_estimated_eps_out_total_graph.gridPos.h }}
+  },
+  "type": "timeseries",
+  "title": "Estimated EPS Out - Selected Total",
+  "transformations": [
+    {
+      "id": "calculateField",
+      "options": {
+        "mode": "reduceRow",
+        "reduce": {
+          "reducer": "sum"
+        },
+        "replaceFields": true,
+        "alias": "Total EPS"
+      }
+    }
+  ],
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 10,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "EPS",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "short",
+      "decimals": 1
+    },
+    "overrides": []
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "right",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
+  "targets": [
+    {
+      "refId": "A",
+      "hide": false,
+      "policy": "default",
+      "resultFormat": "time_series",
+      "orderByTime": "ASC",
+      "tags": [
+        {
+          "key": "host",
+          "value": "/^$servername$/",
+          "operator": "=~"
+        }
+      ],
+      "groupBy": [
+        {
+          "type": "time",
+          "params": [
+            "$__interval"
+          ]
+        },
+        {
+          "type": "tag",
+          "params": [
+            "host"
+          ]
+        },
+        {
+          "type": "fill",
+          "params": [
+            "null"
+          ]
+        }
+      ],
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "out"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "measurement": "logstash_events",
+      "query": "SELECT non_negative_derivative(mean(\"in\"), 1s) FROM \"logstash_events\" WHERE (\"host\" =~ /^$servername$/) AND $timeFilter GROUP BY time($__interval), \"host\" fill(null)",
+      "rawQuery": false
+    }
+  ],
+  "maxDataPoints": null,
+  "description": "",
+  "timeFrom": null,
+  "timeShift": null
+}

salt/grafana/panels/logstash_estimated_eps_stat.json.jinja

@@ -1,112 +0,0 @@
-{
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {
-      "thresholds": {
-        "mode": "absolute",
-        "steps": [
-          {
-            "color": "dark-red",
-            "value": null
-          },
-          {
-            "value": 1,
-            "color": "dark-green"
-          }
-        ]
-      },
-      "mappings": [
-        {
-          "op": "=",
-          "text": "N/A",
-          "value": "null",
-          "$$hashKey": "object:730",
-          "id": 0,
-          "type": 1
-        }
-      ],
-      "unit": "short",
-      "decimals": 0,
-      "color": {
-        "mode": "thresholds"
-      }
-    },
-    "overrides": []
-  },
-  "gridPos": {
-    "x": {{ PANELS.logstash_estimated_eps_stat.gridPos.x }},
-    "y": {{ PANELS.logstash_estimated_eps_stat.gridPos.y }},
-    "w": {{ PANELS.logstash_estimated_eps_stat.gridPos.w }},
-    "h": {{ PANELS.logstash_estimated_eps_stat.gridPos.h }}
-  },
-  "id": 23,
-  "interval": "30s",
-  "links": [],
-  "maxDataPoints": 750,
-  "targets": [
-    {
-      "dsType": "influxdb",
-      "groupBy": [
-        {
-          "params": [
-            "$interval"
-          ],
-          "type": "time"
-        },
-        {
-          "params": [
-            "null"
-          ],
-          "type": "fill"
-        }
-      ],
-      "measurement": "consumptioneps",
-      "orderByTime": "ASC",
-      "policy": "default",
-      "queryType": "randomWalk",
-      "refId": "A",
-      "resultFormat": "time_series",
-      "select": [
-        [
-          {
-            "params": [
-              "eps"
-            ],
-            "type": "field"
-          },
-          {
-            "params": [],
-            "type": "last"
-          }
-        ]
-      ],
-      "tags": [
-        {
-          "key": "host",
-          "operator": "=",
-          "value": "$servername"
-        }
-      ]
-    }
-  ],
-  "title": "Estimated EPS",
-  "type": "stat",
-  "options": {
-    "reduceOptions": {
-      "values": false,
-      "calcs": [
-        "lastNotNull"
-      ],
-      "fields": ""
-    },
-    "orientation": "horizontal",
-    "text": {},
-    "textMode": "value",
-    "colorMode": "value",
-    "graphMode": "area",
-    "justifyMode": "auto"
-  },
-  "cacheTimeout": null,
-  "pluginVersion": "7.5.4",
-  "timeFrom": null
-}

salt/grafana/panels/logstash_indexing_eps_in_out_searchnode_graph.json.jinja

@@ -0,0 +1,411 @@
+{
+  "id": 445554,
+  "gridPos": {
+    "x": {{ PANELS.logstash_indexing_eps_in_out_searchnode_graph.gridPos.x }},
+    "y": {{ PANELS.logstash_indexing_eps_in_out_searchnode_graph.gridPos.y }},
+    "w": {{ PANELS.logstash_indexing_eps_in_out_searchnode_graph.gridPos.w }},
+    "h": {{ PANELS.logstash_indexing_eps_in_out_searchnode_graph.gridPos.h }}
+  },
+  "type": "timeseries",
+  "title": "Indexing Events Per Second - $searchnode",
+  "repeat": "searchnode",
+  "repeatDirection": "v",
+  "transformations": [],
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 50,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "EPS",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "decimals": 2,
+      "unit": "short"
+    },
+    "overrides": [
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/Incoming/"
+        },
+        "properties": [
+          {
+            "id": "color",
+            "value": {
+              "fixedColor": "orange",
+              "mode": "fixed"
+            }
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/Outgoing/"
+        },
+        "properties": [
+          {
+            "id": "color",
+            "value": {
+              "fixedColor": "green",
+              "mode": "fixed"
+            }
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byName",
+          "options": "Incoming hidden"
+        },
+        "properties": [
+          {
+            "id": "custom.fillBelowTo",
+            "value": "Outgoing hidden"
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byName",
+          "options": "Incoming"
+        },
+        "properties": [
+          {
+            "id": "custom.fillOpacity",
+            "value": 0
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byName",
+          "options": "Outgoing"
+        },
+        "properties": [
+          {
+            "id": "custom.fillOpacity",
+            "value": 0
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byName",
+          "options": "Outgoing hidden"
+        },
+        "properties": [
+          {
+            "id": "custom.fillBelowTo",
+            "value": "Incoming hidden"
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/hidden/"
+        },
+        "properties": [
+          {
+            "id": "custom.hideFrom",
+            "value": {
+              "legend": true,
+              "tooltip": true,
+              "viz": false
+            }
+          }
+        ]
+      }
+    ]
+  },
+  "options": {
+    "tooltip": {
+      "mode": "multi"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "bottom",
+      "calcs": [
+        "max",
+        "mean"
+      ]
+    }
+  },
+  "targets": [
+    {
+      "alias": "Incoming",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "host"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "hide": false,
+      "measurement": "logstash_events",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_derivative(mean(\"in\"), 1s) FROM \"logstash_events\" WHERE (\"role\" = \"searchnode\") AND $timeFilter GROUP BY time($__interval), \"host\" fill(null)",
+      "rawQuery": false,
+      "refId": "A",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "in"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "Outgoing",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "host"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "hide": false,
+      "measurement": "logstash_events",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_derivative(mean(\"in\"), 1s) FROM \"logstash_events\" WHERE (\"role\" = \"searchnode\") AND $timeFilter GROUP BY time($__interval), \"host\" fill(null)",
+      "rawQuery": false,
+      "refId": "B",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "out"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "Incoming hidden",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "host"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "hide": false,
+      "measurement": "logstash_events",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_derivative(mean(\"in\"), 1s) FROM \"logstash_events\" WHERE (\"role\" = \"searchnode\") AND $timeFilter GROUP BY time($__interval), \"host\" fill(null)",
+      "rawQuery": false,
+      "refId": "C",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "in"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    },
+    {
+      "alias": "Outgoing hidden",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "host"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "hide": false,
+      "measurement": "logstash_events",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_derivative(mean(\"in\"), 1s) FROM \"logstash_events\" WHERE (\"role\" = \"searchnode\") AND $timeFilter GROUP BY time($__interval), \"host\" fill(null)",
+      "rawQuery": false,
+      "refId": "D",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "out"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "host",
+          "operator": "=",
+          "value": "$searchnode"
+        }
+      ]
+    }
+  ],
+  "description": "",
+  "maxDataPoints": null,
+  "timeFrom": null,
+  "timeShift": null
+}

salt/grafana/panels/logstash_indexing_eps_in_searchnode_total_graph.json.jinja

@@ -0,0 +1,170 @@
+{
+  "id": 69001,
+  "gridPos": {
+    "x": {{ PANELS.logstash_indexing_eps_in_searchnode_total_graph.gridPos.x }},
+    "y": {{ PANELS.logstash_indexing_eps_in_searchnode_total_graph.gridPos.y }},
+    "w": {{ PANELS.logstash_indexing_eps_in_searchnode_total_graph.gridPos.w }},
+    "h": {{ PANELS.logstash_indexing_eps_in_searchnode_total_graph.gridPos.h }}
+  },
+  "type": "timeseries",
+  "title": "Total Searchnode Indexing Events Per Second",
+  "transformations": [
+    {
+      "id": "calculateField",
+      "options": {
+        "alias": "Total EPS",
+        "mode": "reduceRow",
+        "reduce": {
+          "reducer": "sum"
+        },
+        "replaceFields": true
+      }
+    }
+  ],
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "fieldConfig": {
+    "defaults": {
+      "custom": {
+        "drawStyle": "line",
+        "lineInterpolation": "linear",
+        "barAlignment": 0,
+        "lineWidth": 1,
+        "fillOpacity": 10,
+        "gradientMode": "none",
+        "spanNulls": false,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "EPS",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "fixed",
+        "fixedColor": "orange"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "decimals": 2,
+      "unit": "short"
+    },
+    "overrides": []
+  },
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "bottom",
+      "calcs": [
+        "max",
+        "mean"
+      ]
+    }
+  },
+  "targets": [
+    {
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "host"
+          ],
+          "type": "tag"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "hide": false,
+      "measurement": "logstash_events",
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_derivative(mean(\"in\"), 1s) FROM \"logstash_events\" WHERE (\"role\" = \"searchnode\") AND $timeFilter GROUP BY time($__interval), \"host\" fill(null)",
+      "rawQuery": false,
+      "refId": "A",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "type": "field",
+            "params": [
+              "in"
+            ]
+          },
+          {
+            "type": "mean",
+            "params": []
+          },
+          {
+            "type": "non_negative_derivative",
+            "params": [
+              "1s"
+            ]
+          }
+        ]
+      ],
+      "tags": [
+        {
+          "key": "role",
+          "operator": "=~",
+          "value": "/search/"
+        },
+        {
+          "key": "role",
+          "value": "heavynode",
+          "operator": "=",
+          "condition": "OR"
+        },
+        {
+          "key": "role",
+          "value": "standalone",
+          "operator": "=",
+          "condition": "OR"
+        },
+        {
+          "key": "role",
+          "value": "eval",
+          "operator": "=",
+          "condition": "OR"
+        }
+      ]
+    }
+  ],
+  "description": "",
+  "maxDataPoints": null,
+  "timeFrom": null,
+  "timeShift": null
+}

salt/grafana/panels/management_interface_drops_graph.json.jinja

@@ -1,263 +1,282 @@
 {
-  "type": "graph",
+  "id": 61877,
-  "title": "Management Interface Drops",
   "gridPos": {
     "x": {{ PANELS.management_interface_drops_graph.gridPos.x }},
     "y": {{ PANELS.management_interface_drops_graph.gridPos.y }},
     "w": {{ PANELS.management_interface_drops_graph.gridPos.w }},
     "h": {{ PANELS.management_interface_drops_graph.gridPos.h }}
   },
-  "id": 61877,
+  "type": "timeseries",
+  "title": "Management Interface Drops",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "maxDataPoints": 750,
+  "interval": "30s",
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "bottom",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
   "targets": [
     {
-      "refId": "A",
+      "alias": "$tag_host: $tag_interface: $col",
-      "queryType": "randomWalk",
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
       "groupBy": [
         {
-          "type": "time",
           "params": [
             "$__interval"
-          ]
+          ],
+          "type": "time"
         },
         {
-          "type": "fill",
           "params": [
             "null"
-          ]
+          ],
+          "type": "fill"
         }
       ],
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_derivative(mean(drop_in), 1s) as \"in\" FROM \"net\" WHERE host =~ /$servername/ AND interface =~ /$manint/ AND $timeFilter GROUP BY time($__interval), host,interface fill(none)",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "A",
+      "resultFormat": "time_series",
       "select": [
         [
           {
-            "type": "field",
             "params": [
               "value"
-            ]
+            ],
+            "type": "field"
           },
           {
-            "type": "mean",
+            "params": [],
-            "params": []
+            "type": "mean"
           }
         ]
       ],
-      "query": "SELECT non_negative_derivative(mean(drop_in), 1s) as \"in\" FROM \"net\" WHERE host =~ /$servername/ AND interface =~ /$manint/ AND $timeFilter GROUP BY time($__interval), host,interface fill(none)",
+      "tags": []
-      "rawQuery": true,
-      "alias": "$tag_host: $tag_interface: $col"
     },
     {
-      "refId": "B",
+      "alias": "$tag_host: $tag_interface: $col",
-      "hide": false,
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
       "groupBy": [
         {
-          "type": "time",
           "params": [
             "$__interval"
-          ]
+          ],
+          "type": "time"
         },
         {
-          "type": "fill",
           "params": [
             "null"
-          ]
+          ],
+          "type": "fill"
         }
       ],
-      "select": [
+      "hide": false,
-        [
+      "orderByTime": "ASC",
-          {
+      "policy": "default",
-            "type": "field",
-            "params": [
-              "value"
-            ]
-          },
-          {
-            "type": "mean",
-            "params": []
-          }
-        ]
-      ],
       "query": "SELECT non_negative_derivative(mean(drop_out), 1s) as \"out\" FROM \"net\" WHERE host =~ /$servername/ AND interface =~ /$manint/ AND $timeFilter GROUP BY time($__interval), host,interface fill(none)",
       "rawQuery": true,
-      "alias": "$tag_host: $tag_interface: $col"
+      "refId": "B",
-    },
-    {
-      "refId": "C",
-      "queryType": "randomWalk",
-      "policy": "default",
       "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
-      "groupBy": [
-        {
-          "type": "time",
-          "params": [
-            "$__interval"
-          ]
-        },
-        {
-          "type": "fill",
-          "params": [
-            "null"
-          ]
-        }
-      ],
       "select": [
         [
           {
-            "type": "field",
             "params": [
               "value"
-            ]
+            ],
+            "type": "field"
           },
           {
-            "type": "mean",
+            "params": [],
-            "params": []
+            "type": "mean"
           }
         ]
       ],
-      "query": "SELECT non_negative_derivative(mean(mean_drop_in), 1s) as \"trend_in\" FROM \"so_long_term\".\"net\" WHERE host =~ /$servername/ AND interface =~ /$manint/ AND $timeFilter GROUP BY time($__interval), host,interface fill(none)",
+      "tags": []
-      "rawQuery": true,
+    },
+    {
       "alias": "$tag_host: $tag_interface: $col",
-      "hide": false
-    },
-    {
-      "refId": "D",
-      "hide": false,
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
       "groupBy": [
         {
-          "type": "time",
           "params": [
             "$__interval"
-          ]
+          ],
+          "type": "time"
         },
         {
-          "type": "fill",
           "params": [
             "null"
-          ]
+          ],
+          "type": "fill"
         }
       ],
+      "hide": false,
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_derivative(mean(mean_drop_in), 1s) as \"trend_in\" FROM \"so_long_term\".\"net\" WHERE host =~ /$servername/ AND interface =~ /$manint/ AND $timeFilter GROUP BY time($__interval), host,interface fill(none)",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "C",
+      "resultFormat": "time_series",
       "select": [
         [
           {
-            "type": "field",
             "params": [
               "value"
-            ]
+            ],
+            "type": "field"
           },
           {
-            "type": "mean",
+            "params": [],
-            "params": []
+            "type": "mean"
           }
         ]
       ],
+      "tags": []
+    },
+    {
+      "alias": "$tag_host: $tag_interface: $col",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "hide": false,
+      "orderByTime": "ASC",
+      "policy": "default",
       "query": "SELECT non_negative_derivative(mean(mean_drop_out), 1s) as \"trend_out\" FROM \"so_long_term\".\"net\" WHERE host =~ /$servername/ AND interface =~ /$manint/ AND $timeFilter GROUP BY time($__interval), host,interface fill(none)",
       "rawQuery": true,
-      "alias": "$tag_host: $tag_interface: $col"
+      "refId": "D",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "value"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
+      "tags": []
     }
   ],
-  "options": {
-    "alertThreshold": true
-  },
-  "datasource": "InfluxDB",
   "fieldConfig": {
-    "defaults": {},
+    "defaults": {
-    "overrides": []
+      "custom": {
-  },
+        "drawStyle": "line",
-  "pluginVersion": "7.5.4",
+        "lineInterpolation": "linear",
-  "renderer": "flot",
+        "barAlignment": 0,
-  "yaxes": [
+        "lineWidth": 1,
-    {
+        "fillOpacity": 10,
-      "label": "Drops per second",
+        "gradientMode": "none",
-      "show": true,
+        "spanNulls": false,
-      "logBase": 1,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "Drops per second",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "pps",
       "min": 0,
-      "max": null,
+      "decimals": 1
-      "format": "pps",
-      "$$hashKey": "object:500"
     },
-    {
+    "overrides": [
-      "label": null,
+      {
-      "show": true,
+        "matcher": {
-      "logBase": 1,
+          "id": "byRegexp",
-      "min": null,
+          "options": "/trend/"
-      "max": null,
+        },
-      "format": "short",
+        "properties": [
-      "$$hashKey": "object:501"
+          {
-    }
+            "id": "custom.fillOpacity",
-  ],
+            "value": 0
-  "xaxis": {
+          },
-    "show": true,
+          {
-    "mode": "time",
+            "id": "custom.lineWidth",
-    "name": null,
+            "value": 4
-    "values": [],
+          },
-    "buckets": null
+          {
+            "id": "custom.lineStyle",
+            "value": {
+              "fill": "dash",
+              "dash": [
+                4,
+                10
+              ]
+            }
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/veth/"
+        },
+        "properties": [
+          {
+            "id": "custom.hideFrom",
+            "value": {
+              "tooltip": true,
+              "viz": true,
+              "legend": true
+            }
+          }
+        ]
+      }
+    ]
   },
-  "yaxis": {
+  "timeFrom": null,
-    "align": false,
+  "timeShift": null
-    "alignLevel": null
-  },
-  "lines": true,
-  "fill": 1,
-  "linewidth": 1,
-  "dashLength": 10,
-  "spaceLength": 10,
-  "pointradius": 2,
-  "legend": {
-    "show": true,
-    "values": true,
-    "min": false,
-    "max": true,
-    "current": true,
-    "total": false,
-    "avg": true,
-    "alignAsTable": true
-  },
-  "nullPointMode": "connected",
-  "tooltip": {
-    "value_type": "individual",
-    "shared": true,
-    "sort": 0
-  },
-  "aliasColors": {},
-  "seriesOverrides": [
-    {
-      "$$hashKey": "object:592",
-      "alias": "/veth/",
-      "hiddenSeries": true,
-      "legend": false
-    },
-    {
-      "$$hashKey": "object:621",
-      "alias": "/trend/",
-      "fill": 0,
-      "linewidth": 4,
-      "dashes": true,
-      "dashLength": 4
-    }
-  ],
-  "thresholds": [],
-  "timeRegions": [],
-  "fillGradient": 0,
-  "dashes": false,
-  "hiddenSeries": false,
-  "points": false,
-  "bars": false,
-  "stack": false,
-  "percentage": false,
-  "steppedLine": false,
-  "decimals": 0,
-  "maxDataPoints": 750,
-  "interval": "30s"
 }

salt/grafana/panels/management_interface_drops_inbound_graph.json.jinja

@@ -1,51 +1,100 @@
 {
-  "aliasColors": {},
+  "id": 61877,
-  "dashLength": 10,
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
   "gridPos": {
     "x": {{ PANELS.management_interface_drops_inbound_graph.gridPos.x }},
     "y": {{ PANELS.management_interface_drops_inbound_graph.gridPos.y }},
     "w": {{ PANELS.management_interface_drops_inbound_graph.gridPos.w }},
     "h": {{ PANELS.management_interface_drops_inbound_graph.gridPos.h }}
   },
-  "id": 61877,
+  "type": "timeseries",
+  "title": "Management Interface Drops - Inbound",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
   "interval": "30s",
-  "legend": {
+  "fieldConfig": {
-    "alignAsTable": true,
+    "defaults": {
-    "avg": true,
+      "custom": {
-    "current": true,
+        "drawStyle": "line",
-    "max": false,
+        "lineInterpolation": "linear",
-    "min": false,
+        "barAlignment": 0,
-    "rightSide": true,
+        "lineWidth": 1,
-    "show": true,
+        "fillOpacity": 0,
-    "sort": "current",
+        "gradientMode": "none",
-    "sortDesc": true,
+        "spanNulls": false,
-    "total": false,
+        "showPoints": "never",
-    "values": true
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "Drops per second",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "pps",
+      "min": 0,
+      "decimals": 1
+    },
+    "overrides": [
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/veth/"
+        },
+        "properties": [
+          {
+            "id": "custom.hideFrom",
+            "value": {
+              "tooltip": true,
+              "viz": true,
+              "legend": true
+            }
+          }
+        ]
+      }
+    ]
   },
-  "lines": true,
-  "linewidth": 1,
-  "maxDataPoints": 750,
-  "nullPointMode": "connected",
   "options": {
-    "alertThreshold": false
+    "tooltip": {
-  },
+      "mode": "single"
-  "pluginVersion": "7.5.4",
+    },
-  "pointradius": 2,
+    "legend": {
-  "renderer": "flot",
+      "displayMode": "table",
-  "seriesOverrides": [
+      "placement": "right",
-    {
+      "calcs": [
-      "$$hashKey": "object:592",
+        "max",
-      "alias": "/veth/",
+        "mean",
-      "hiddenSeries": true,
+        "lastNotNull"
-      "legend": false
+      ]
     }
-  ],
+  },
-  "spaceLength": 10,
   "targets": [
     {
       "alias": "$tag_host: $tag_role",
@@ -87,57 +136,7 @@
       "tags": []
     }
   ],
-  "thresholds": [],
+  "maxDataPoints": null,
-  "timeRegions": [],
-  "title": "Management Interface Drops - Inbound",
-  "tooltip": {
-    "shared": true,
-    "sort": 2,
-    "value_type": "individual"
-  },
-  "type": "graph",
-  "xaxis": {
-    "buckets": null,
-    "mode": "time",
-    "name": null,
-    "show": true,
-    "values": []
-  },
-  "yaxes": [
-    {
-      "$$hashKey": "object:500",
-      "format": "pps",
-      "label": "Drops per second",
-      "logBase": 1,
-      "max": null,
-      "min": 0,
-      "show": true
-    },
-    {
-      "$$hashKey": "object:501",
-      "format": "short",
-      "label": null,
-      "logBase": 1,
-      "max": null,
-      "min": null,
-      "show": true,
-      "decimals": 0
-    }
-  ],
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "fill": 0,
-  "bars": false,
-  "dashes": false,
-  "decimals": 0,
-  "fillGradient": 0,
-  "hiddenSeries": false,
-  "percentage": false,
-  "points": false,
-  "stack": false,
-  "steppedLine": false,
   "timeFrom": null,
   "timeShift": null
 }

salt/grafana/panels/management_interface_drops_outbound_graph.json.jinja

@@ -1,51 +1,100 @@
 {
-  "aliasColors": {},
+  "id": 188189,
-  "dashLength": 10,
-  "datasource": "InfluxDB",
-  "fieldConfig": {
-    "defaults": {},
-    "overrides": []
-  },
   "gridPos": {
     "x": {{ PANELS.management_interface_drops_outbound_graph.gridPos.x }},
     "y": {{ PANELS.management_interface_drops_outbound_graph.gridPos.y }},
     "w": {{ PANELS.management_interface_drops_outbound_graph.gridPos.w }},
     "h": {{ PANELS.management_interface_drops_outbound_graph.gridPos.h }}
   },
-  "id": 188189,
+  "type": "timeseries",
+  "title": "Management Interface Drops - Outbound",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
   "interval": "30s",
-  "legend": {
+  "fieldConfig": {
-    "alignAsTable": true,
+    "defaults": {
-    "avg": true,
+      "custom": {
-    "current": true,
+        "drawStyle": "line",
-    "max": false,
+        "lineInterpolation": "linear",
-    "min": false,
+        "barAlignment": 0,
-    "rightSide": true,
+        "lineWidth": 1,
-    "show": true,
+        "fillOpacity": 0,
-    "sort": "current",
+        "gradientMode": "none",
-    "sortDesc": true,
+        "spanNulls": false,
-    "total": false,
+        "showPoints": "never",
-    "values": true
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "Drops per second",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "value": null,
+            "color": "green"
+          },
+          {
+            "value": 80,
+            "color": "red"
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "pps",
+      "min": 0,
+      "decimals": 1
+    },
+    "overrides": [
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/veth/"
+        },
+        "properties": [
+          {
+            "id": "custom.hideFrom",
+            "value": {
+              "tooltip": true,
+              "viz": true,
+              "legend": true
+            }
+          }
+        ]
+      }
+    ]
   },
-  "lines": true,
-  "linewidth": 1,
-  "maxDataPoints": 750,
-  "nullPointMode": "connected",
   "options": {
-    "alertThreshold": false
+    "tooltip": {
-  },
+      "mode": "single"
-  "pluginVersion": "7.5.4",
+    },
-  "pointradius": 2,
+    "legend": {
-  "renderer": "flot",
+      "displayMode": "table",
-  "seriesOverrides": [
+      "placement": "right",
-    {
+      "calcs": [
-      "$$hashKey": "object:592",
+        "max",
-      "alias": "/veth/",
+        "mean",
-      "hiddenSeries": true,
+        "lastNotNull"
-      "legend": false
+      ]
     }
-  ],
+  },
-  "spaceLength": 10,
   "targets": [
     {
       "alias": "$tag_host: $tag_role",
@@ -87,57 +136,7 @@
       "tags": []
     }
   ],
-  "thresholds": [],
+  "maxDataPoints": null,
-  "timeRegions": [],
-  "title": "Management Interface Drops - Outbound",
-  "tooltip": {
-    "shared": true,
-    "sort": 2,
-    "value_type": "individual"
-  },
-  "type": "graph",
-  "xaxis": {
-    "buckets": null,
-    "mode": "time",
-    "name": null,
-    "show": true,
-    "values": []
-  },
-  "yaxes": [
-    {
-      "$$hashKey": "object:500",
-      "format": "pps",
-      "label": "Drops per second",
-      "logBase": 1,
-      "max": null,
-      "min": 0,
-      "show": true
-    },
-    {
-      "$$hashKey": "object:501",
-      "format": "short",
-      "label": null,
-      "logBase": 1,
-      "max": null,
-      "min": null,
-      "show": true,
-      "decimals": 0
-    }
-  ],
-  "yaxis": {
-    "align": false,
-    "alignLevel": null
-  },
-  "fill": 0,
-  "bars": false,
-  "dashes": false,
-  "decimals": 0,
-  "fillGradient": 0,
-  "hiddenSeries": false,
-  "percentage": false,
-  "points": false,
-  "stack": false,
-  "steppedLine": false,
   "timeFrom": null,
   "timeShift": null
 }

salt/grafana/panels/management_interface_packets_graph.json.jinja

@@ -1,262 +1,283 @@
 {
-  "type": "graph",
+  "id": 61875,
-  "title": "Management Interface Packets",
   "gridPos": {
     "x": {{ PANELS.management_interface_packets_graph.gridPos.x }},
     "y": {{ PANELS.management_interface_packets_graph.gridPos.y }},
     "w": {{ PANELS.management_interface_packets_graph.gridPos.w }},
     "h": {{ PANELS.management_interface_packets_graph.gridPos.h }}
   },
-  "id": 61875,
+  "type": "timeseries",
+  "title": "Management Interface Packets",
+  "datasource": "InfluxDB",
+  "pluginVersion": "8.2.1",
+  "interval": "30s",
+  "options": {
+    "tooltip": {
+      "mode": "single"
+    },
+    "legend": {
+      "displayMode": "table",
+      "placement": "bottom",
+      "calcs": [
+        "max",
+        "mean",
+        "lastNotNull"
+      ]
+    }
+  },
   "targets": [
     {
-      "refId": "A",
-      "queryType": "randomWalk",
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
-      "groupBy": [
-        {
-          "type": "time",
-          "params": [
-            "$__interval"
-          ]
-        },
-        {
-          "type": "fill",
-          "params": [
-            "null"
-          ]
-        }
-      ],
-      "select": [
-        [
-          {
-            "type": "field",
-            "params": [
-              "value"
-            ]
-          },
-          {
-            "type": "mean",
-            "params": []
-          }
-        ]
-      ],
-      "query": "SELECT non_negative_derivative(mean(packets_recv), 1s) as \"in\" FROM \"net\" WHERE host =~ /$servername/ AND interface =~ /$manint/ AND $timeFilter GROUP BY time($__interval), * fill(none)",
-      "rawQuery": true,
-      "alias": "$tag_host: $tag_interface: $col"
-    },
-    {
-      "refId": "B",
-      "hide": false,
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
-      "groupBy": [
-        {
-          "type": "time",
-          "params": [
-            "$__interval"
-          ]
-        },
-        {
-          "type": "fill",
-          "params": [
-            "null"
-          ]
-        }
-      ],
-      "select": [
-        [
-          {
-            "type": "field",
-            "params": [
-              "value"
-            ]
-          },
-          {
-            "type": "mean",
-            "params": []
-          }
-        ]
-      ],
-      "query": "SELECT non_negative_derivative(mean(packets_sent), 1s) as \"out\" FROM \"net\" WHERE host =~ /$servername/ AND interface =~ /$manint/ AND $timeFilter GROUP BY time($__interval), * fill(none)",
-      "rawQuery": true,
-      "alias": "$tag_host: $tag_interface: $col"
-    },
-    {
-      "refId": "C",
-      "queryType": "randomWalk",
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
-      "groupBy": [
-        {
-          "type": "time",
-          "params": [
-            "$__interval"
-          ]
-        },
-        {
-          "type": "fill",
-          "params": [
-            "null"
-          ]
-        }
-      ],
-      "select": [
-        [
-          {
-            "type": "field",
-            "params": [
-              "value"
-            ]
-          },
-          {
-            "type": "mean",
-            "params": []
-          }
-        ]
-      ],
-      "query": "SELECT non_negative_derivative(mean(mean_packets_recv), 1s) as \"trend_in\" FROM \"so_long_term\".\"net\" WHERE host =~ /$servername/ AND interface =~ /$manint/ AND $timeFilter GROUP BY time($__interval), * fill(none)",
-      "rawQuery": true,
       "alias": "$tag_host: $tag_interface: $col",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_derivative(mean(packets_recv), 1s) as \"in\" FROM \"net\" WHERE host =~ /$servername/ AND interface =~ /$manint/ AND $timeFilter GROUP BY time($__interval), * fill(none)",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "A",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "value"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
+      "tags": [],
       "hide": false
     },
     {
-      "refId": "D",
+      "alias": "$tag_host: $tag_interface: $col",
-      "hide": false,
-      "policy": "default",
-      "resultFormat": "time_series",
-      "orderByTime": "ASC",
-      "tags": [],
       "groupBy": [
         {
-          "type": "time",
           "params": [
             "$__interval"
-          ]
+          ],
+          "type": "time"
         },
         {
-          "type": "fill",
           "params": [
             "null"
-          ]
+          ],
+          "type": "fill"
         }
       ],
+      "hide": false,
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_derivative(mean(packets_sent), 1s) as \"out\" FROM \"net\" WHERE host =~ /$servername/ AND interface =~ /$manint/ AND $timeFilter GROUP BY time($__interval), * fill(none)",
+      "rawQuery": true,
+      "refId": "B",
+      "resultFormat": "time_series",
       "select": [
         [
           {
-            "type": "field",
             "params": [
               "value"
-            ]
+            ],
+            "type": "field"
           },
           {
-            "type": "mean",
+            "params": [],
-            "params": []
+            "type": "mean"
           }
         ]
       ],
+      "tags": []
+    },
+    {
+      "alias": "$tag_host: $tag_interface: $col",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "hide": false,
+      "orderByTime": "ASC",
+      "policy": "default",
+      "query": "SELECT non_negative_derivative(mean(mean_packets_recv), 1s) as \"trend_in\" FROM \"so_long_term\".\"net\" WHERE host =~ /$servername/ AND interface =~ /$manint/ AND $timeFilter GROUP BY time($__interval), * fill(none)",
+      "queryType": "randomWalk",
+      "rawQuery": true,
+      "refId": "C",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "value"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
+      "tags": []
+    },
+    {
+      "alias": "$tag_host: $tag_interface: $col",
+      "groupBy": [
+        {
+          "params": [
+            "$__interval"
+          ],
+          "type": "time"
+        },
+        {
+          "params": [
+            "null"
+          ],
+          "type": "fill"
+        }
+      ],
+      "hide": false,
+      "orderByTime": "ASC",
+      "policy": "default",
       "query": "SELECT non_negative_derivative(mean(mean_packets_sent), 1s) as \"trend_out\" FROM \"so_long_term\".\"net\" WHERE host =~ /$servername/ AND interface =~ /$manint/ AND $timeFilter GROUP BY time($__interval), * fill(none)",
       "rawQuery": true,
-      "alias": "$tag_host: $tag_interface: $col"
+      "refId": "D",
+      "resultFormat": "time_series",
+      "select": [
+        [
+          {
+            "params": [
+              "value"
+            ],
+            "type": "field"
+          },
+          {
+            "params": [],
+            "type": "mean"
+          }
+        ]
+      ],
+      "tags": []
     }
   ],
-  "options": {
-    "alertThreshold": true
-  },
-  "datasource": "InfluxDB",
   "fieldConfig": {
-    "defaults": {},
+    "defaults": {
-    "overrides": []
+      "custom": {
-  },
+        "drawStyle": "line",
-  "pluginVersion": "7.5.4",
+        "lineInterpolation": "linear",
-  "renderer": "flot",
+        "barAlignment": 0,
-  "yaxes": [
+        "lineWidth": 1,
-    {
+        "fillOpacity": 10,
-      "label": "Packets per second",
+        "gradientMode": "none",
-      "show": true,
+        "spanNulls": false,
-      "logBase": 1,
+        "showPoints": "never",
+        "pointSize": 5,
+        "stacking": {
+          "mode": "none",
+          "group": "A"
+        },
+        "axisPlacement": "auto",
+        "axisLabel": "Packets per second",
+        "scaleDistribution": {
+          "type": "linear"
+        },
+        "hideFrom": {
+          "tooltip": false,
+          "viz": false,
+          "legend": false
+        },
+        "thresholdsStyle": {
+          "mode": "off"
+        }
+      },
+      "color": {
+        "mode": "palette-classic"
+      },
+      "thresholds": {
+        "mode": "absolute",
+        "steps": [
+          {
+            "color": "green",
+            "value": null
+          }
+        ]
+      },
+      "mappings": [],
+      "unit": "pps",
       "min": 0,
-      "max": null,
+      "decimals": 1
-      "format": "pps",
-      "$$hashKey": "object:241"
     },
-    {
+    "overrides": [
-      "label": null,
+      {
-      "show": true,
+        "matcher": {
-      "logBase": 1,
+          "id": "byRegexp",
-      "min": null,
+          "options": "/trend/"
-      "max": null,
+        },
-      "format": "short",
+        "properties": [
-      "$$hashKey": "object:242"
+          {
-    }
+            "id": "custom.fillOpacity",
-  ],
+            "value": 0
-  "xaxis": {
+          },
-    "show": true,
+          {
-    "mode": "time",
+            "id": "custom.lineWidth",
-    "name": null,
+            "value": 4
-    "values": [],
+          },
-    "buckets": null
+          {
+            "id": "custom.lineStyle",
+            "value": {
+              "fill": "dash",
+              "dash": [
+                4,
+                10
+              ]
+            }
+          }
+        ]
+      },
+      {
+        "matcher": {
+          "id": "byRegexp",
+          "options": "/veth/"
+        },
+        "properties": [
+          {
+            "id": "custom.hideFrom",
+            "value": {
+              "tooltip": true,
+              "viz": true,
+              "legend": true
+            }
+          }
+        ]
+      }
+    ]
   },
-  "yaxis": {
+  "maxDataPoints": null,
-    "align": false,
+  "timeFrom": null,
-    "alignLevel": null
+  "timeShift": null
-  },
-  "lines": true,
-  "fill": 1,
-  "linewidth": 1,
-  "dashLength": 10,
-  "spaceLength": 10,
-  "pointradius": 2,
-  "legend": {
-    "show": true,
-    "values": true,
-    "min": false,
-    "max": true,
-    "current": true,
-    "total": false,
-    "avg": true,
-    "alignAsTable": true
-  },
-  "nullPointMode": "connected",
-  "tooltip": {
-    "value_type": "individual",
-    "shared": true,
-    "sort": 0
-  },
-  "aliasColors": {},
-  "seriesOverrides": [
-    {
-      "$$hashKey": "object:413",
-      "alias": "/veth/",
-      "hiddenSeries": true,
-      "legend": false
-    },
-    {
-      "$$hashKey": "object:442",
-      "alias": "/trend/",
-      "fill": 0,
-      "linewidth": 4,
-      "dashes": true,
-      "dashLength": 4
-    }
-  ],
-  "thresholds": [],
-  "timeRegions": [],
-  "fillGradient": 0,
-  "dashes": false,
-  "hiddenSeries": false,
-  "points": false,
-  "bars": false,
-  "stack": false,
-  "percentage": false,
-  "steppedLine": false,
-  "maxDataPoints": 750,
-  "interval": "30s"
 }