From 216869859563f3025a8313f2f7f0e416ec7b79e0 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 22 Jan 2024 20:27:19 -0500 Subject: [PATCH 01/80] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9cf89c6c7..7d52aac7f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.50 +2.4.0-foxtrot From 2ecac38f6d4aba6c5e1cc5bd31ef48209a098a80 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Jun 2024 13:50:29 -0400 Subject: [PATCH 02/80] disable logstash on heavynodes --- salt/manager/tools/sbin/so-minion | 1 - salt/manager/tools/sbin/soup | 17 ++++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/so-minion b/salt/manager/tools/sbin/so-minion index da1a6d2a2..bdcf4a08d 100755 --- a/salt/manager/tools/sbin/so-minion +++ b/salt/manager/tools/sbin/so-minion @@ -528,7 +528,6 @@ function createHEAVYNODE() { pcapspace add_elasticsearch_to_minion add_elastic_agent_to_minion - add_logstash_to_minion add_sensor_to_minion add_strelka_to_minion add_redis_to_minion diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 9ee4058f9..61246c29a 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -308,6 +308,21 @@ clone_to_tmp() { fi } +disable_logstash_heavynodes() { + c=0 + printf "\nChecking for heavynodes and disabling Logstash if they exist\n" + for file in /opt/so/saltstack/local/pillar/minions/*.sls; do + if [[ $file =~ "_heavynode.sls" && ! $file =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then + if [ $c -eq 0 ]; then + ((c++)) + FINAL_MESSAGE_QUEUE+=("Logstash has been disabled on all heavynodes. It can be re-enabled via Grid Configuration in SOC.") + fi + echo "Disabling Logstash for: $file" + so-yaml.py replace "$file" logstash.enabled False + fi + done +} + enable_highstate() { echo "Enabling highstate." salt-call state.enable highstate -l info --local @@ -477,7 +492,7 @@ post_to_2.4.70() { } post_to_2.4.80() { - echo "Nothing to apply" + disable_logstash_heavynodes POSTVERSION=2.4.80 } From 88fc1bbe329f8ffc66a282255e194fda1e6b1dac Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 11 Jun 2024 16:32:57 -0400 Subject: [PATCH 03/80] quotes on vars --- salt/manager/tools/sbin/soup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 61246c29a..06ce1757b 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -312,8 +312,8 @@ disable_logstash_heavynodes() { c=0 printf "\nChecking for heavynodes and disabling Logstash if they exist\n" for file in /opt/so/saltstack/local/pillar/minions/*.sls; do - if [[ $file =~ "_heavynode.sls" && ! $file =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then - if [ $c -eq 0 ]; then + if [[ "$file" =~ "_heavynode.sls" && ! "$file" =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then + if [ "$c" -eq 0 ]; then ((c++)) FINAL_MESSAGE_QUEUE+=("Logstash has been disabled on all heavynodes. It can be re-enabled via Grid Configuration in SOC.") fi From 98837bc3790964db831a5dc95994f0eea075546f Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 12 Jun 2024 09:11:02 -0400 Subject: [PATCH 04/80] this method does not cause soup to fail --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 06ce1757b..e29a6811e 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -314,7 +314,7 @@ disable_logstash_heavynodes() { for file in /opt/so/saltstack/local/pillar/minions/*.sls; do if [[ "$file" =~ "_heavynode.sls" && ! "$file" =~ "/opt/so/saltstack/local/pillar/minions/adv_" ]]; then if [ "$c" -eq 0 ]; then - ((c++)) + c=$((c + 1)) FINAL_MESSAGE_QUEUE+=("Logstash has been disabled on all heavynodes. It can be re-enabled via Grid Configuration in SOC.") fi echo "Disabling Logstash for: $file" From 55f8303dc29f59225d126237e611430d1154721d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 17 Jun 2024 10:06:43 -0400 Subject: [PATCH 05/80] remove manager and search pipelines from heavynode --- salt/logstash/defaults.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index 348acb622..75abf80f6 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -7,9 +7,7 @@ logstash: - search receiver: - receiver - heavynode: - - manager - - search + heavynode: [] searchnode: - search manager: From 81fcd68e9b3cb5ebfe0402e41ec0af03f17b4c2d Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 20 Jun 2024 16:42:11 -0400 Subject: [PATCH 06/80] create and use redis:nodes and elasticsearch:nodes pillars --- pillar/elasticsearch/nodes.sls | 34 +++++++++++++++++++ pillar/logstash/nodes.sls | 9 +++-- pillar/redis/nodes.sls | 34 +++++++++++++++++++ pillar/top.sls | 6 ++++ salt/elasticsearch/config.map.jinja | 23 +++++++++---- salt/elasticsearch/enabled.sls | 8 ++--- .../so-elasticsearch-cluster-settings | 2 +- salt/logstash/map.jinja | 17 ++++++---- .../config/so/0900_input_redis.conf.jinja | 6 ++-- salt/soc/defaults.map.jinja | 2 +- salt/soc/merged.map.jinja | 3 +- 11 files changed, 114 insertions(+), 30 deletions(-) create mode 100644 pillar/elasticsearch/nodes.sls create mode 100644 pillar/redis/nodes.sls diff --git a/pillar/elasticsearch/nodes.sls b/pillar/elasticsearch/nodes.sls new file mode 100644 index 000000000..588b51217 --- /dev/null +++ b/pillar/elasticsearch/nodes.sls @@ -0,0 +1,34 @@ +{% set node_types = {} %} +{% for minionid, ip in salt.saltutil.runner( + 'mine.get', + tgt='elasticsearch:enabled:true', + fun='network.ip_addrs', + tgt_type='pillar') | dictsort() +%} + +# only add a node to the pillar if it returned an ip from the mine +{% if ip | length > 0%} +{% set hostname = minionid.split('_') | first %} +{% set node_type = minionid.split('_') | last %} +{% if node_type not in node_types.keys() %} +{% do node_types.update({node_type: {hostname: ip[0]}}) %} +{% else %} +{% if hostname not in node_types[node_type] %} +{% do node_types[node_type].update({hostname: ip[0]}) %} +{% else %} +{% do node_types[node_type][hostname].update(ip[0]) %} +{% endif %} +{% endif %} +{% endif %} +{% endfor %} + + +elasticsearch: + nodes: +{% for node_type, values in node_types.items() %} + {{node_type}}: +{% for hostname, ip in values.items() %} + {{hostname}}: + ip: {{ip}} +{% endfor %} +{% endfor %} diff --git a/pillar/logstash/nodes.sls b/pillar/logstash/nodes.sls index a77978821..4cea7737a 100644 --- a/pillar/logstash/nodes.sls +++ b/pillar/logstash/nodes.sls @@ -1,16 +1,15 @@ {% set node_types = {} %} -{% set cached_grains = salt.saltutil.runner('cache.grains', tgt='*') %} {% for minionid, ip in salt.saltutil.runner( 'mine.get', - tgt='G@role:so-manager or G@role:so-managersearch or G@role:so-standalone or G@role:so-searchnode or G@role:so-heavynode or G@role:so-receiver or G@role:so-fleet ', + tgt='logstash:enabled:true', fun='network.ip_addrs', - tgt_type='compound') | dictsort() + tgt_type='pillar') | dictsort() %} # only add a node to the pillar if it returned an ip from the mine {% if ip | length > 0%} -{% set hostname = cached_grains[minionid]['host'] %} -{% set node_type = minionid.split('_')[1] %} +{% set hostname = minionid.split('_') | first %} +{% set node_type = minionid.split('_') | last %} {% if node_type not in node_types.keys() %} {% do node_types.update({node_type: {hostname: ip[0]}}) %} {% else %} diff --git a/pillar/redis/nodes.sls b/pillar/redis/nodes.sls new file mode 100644 index 000000000..618b31715 --- /dev/null +++ b/pillar/redis/nodes.sls @@ -0,0 +1,34 @@ +{% set node_types = {} %} +{% for minionid, ip in salt.saltutil.runner( + 'mine.get', + tgt='redis:enabled:true', + fun='network.ip_addrs', + tgt_type='pillar') | dictsort() +%} + +# only add a node to the pillar if it returned an ip from the mine +{% if ip | length > 0%} +{% set hostname = minionid.split('_') | first %} +{% set node_type = minionid.split('_') | last %} +{% if node_type not in node_types.keys() %} +{% do node_types.update({node_type: {hostname: ip[0]}}) %} +{% else %} +{% if hostname not in node_types[node_type] %} +{% do node_types[node_type].update({hostname: ip[0]}) %} +{% else %} +{% do node_types[node_type][hostname].update(ip[0]) %} +{% endif %} +{% endif %} +{% endif %} +{% endfor %} + + +redis: + nodes: +{% for node_type, values in node_types.items() %} + {{node_type}}: +{% for hostname, ip in values.items() %} + {{hostname}}: + ip: {{ip}} +{% endfor %} +{% endfor %} diff --git a/pillar/top.sls b/pillar/top.sls index b6d6c2e73..d75d3e3a7 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -47,10 +47,12 @@ base: - kibana.adv_kibana - kratos.soc_kratos - kratos.adv_kratos + - redis.nodes - redis.soc_redis - redis.adv_redis - influxdb.soc_influxdb - influxdb.adv_influxdb + - elasticsearch.nodes - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch - elasticfleet.soc_elasticfleet @@ -144,10 +146,12 @@ base: - idstools.adv_idstools - kratos.soc_kratos - kratos.adv_kratos + - redis.nodes - redis.soc_redis - redis.adv_redis - influxdb.soc_influxdb - influxdb.adv_influxdb + - elasticsearch.nodes - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch - elasticfleet.soc_elasticfleet @@ -209,11 +213,13 @@ base: - logstash.nodes - logstash.soc_logstash - logstash.adv_logstash + - elasticsearch.nodes - elasticsearch.soc_elasticsearch - elasticsearch.adv_elasticsearch {% if salt['file.file_exists']('/opt/so/saltstack/local/pillar/elasticsearch/auth.sls') %} - elasticsearch.auth {% endif %} + - redis.nodes - redis.soc_redis - redis.adv_redis - minions.{{ grains.id }} diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 4e57199af..8127df848 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -3,21 +3,30 @@ {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} -{# ES_LOGSTASH_NODES is the same as LOGSTASH_NODES from logstash/map.jinja but heavynodes and fleet nodes are removed #} -{% set ES_LOGSTASH_NODES = [] %} -{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} +{# this is a list of dicts containing hostname:ip for elasticsearch nodes that need to know about each other for cluster #} +{% set ELASTICSEARCH_SEED_HOSTS = [] %} +{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} {% for node_type, node_details in node_data.items() | sort %} -{% if node_type not in ['heavynode', 'fleet'] %} +{% if node_type != 'heavynode' %} {% for hostname in node_data[node_type].keys() %} -{% do ES_LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %} +{% do ELASTICSEARCH_SEED_HOSTS.append({hostname:node_details[hostname].ip}) %} {% endfor %} {% endif %} {% endfor %} +{# this is a list of dicts containing hostname:ip of all nodes running elasticsearch #} +{% set ELASTICSEARCH_NODES = [] %} +{% set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} +{% for node_type, node_details in node_data.items() %} +{% for hostname in node_data[node_type].keys() %} +{% do ELASTICSEARCH_NODES.append({hostname:node_details[hostname].ip}) %} +{% endfor %} +{% endfor %} + {% if grains.id.split('_') | last in ['manager','managersearch','standalone'] %} - {% if ES_LOGSTASH_NODES | length > 1 %} + {% if ELASTICSEARCH_SEED_HOSTS | length > 1 %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.update({'discovery': {'seed_hosts': []}}) %} - {% for NODE in ES_LOGSTASH_NODES %} + {% for NODE in ELASTICSEARCH_SEED_HOSTS %} {% do ELASTICSEARCHDEFAULTS.elasticsearch.config.discovery.seed_hosts.append(NODE.keys()|first) %} {% endfor %} {% endif %} diff --git a/salt/elasticsearch/enabled.sls b/salt/elasticsearch/enabled.sls index 1d35d3505..383fd1cb4 100644 --- a/salt/elasticsearch/enabled.sls +++ b/salt/elasticsearch/enabled.sls @@ -7,8 +7,8 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'docker/docker.map.jinja' import DOCKER %} -{% from 'logstash/map.jinja' import LOGSTASH_NODES %} -{% from 'elasticsearch/config.map.jinja' import ES_LOGSTASH_NODES %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_SEED_HOSTS %} {% from 'elasticsearch/config.map.jinja' import ELASTICSEARCHMERGED %} {% set TEMPLATES = salt['pillar.get']('elasticsearch:templates', {}) %} {% from 'elasticsearch/template.map.jinja' import ES_INDEX_SETTINGS %} @@ -27,7 +27,7 @@ so-elasticsearch: - sobridge: - ipv4_address: {{ DOCKER.containers['so-elasticsearch'].ip }} - extra_hosts: - {% for node in LOGSTASH_NODES %} + {% for node in ELASTICSEARCH_NODES %} {% for hostname, ip in node.items() %} - {{hostname}}:{{ip}} {% endfor %} @@ -38,7 +38,7 @@ so-elasticsearch: {% endfor %} {% endif %} - environment: - {% if ES_LOGSTASH_NODES | length == 1 or GLOBALS.role == 'so-heavynode' %} + {% if ELASTICSEARCH_SEED_HOSTS | length == 1 or GLOBALS.role == 'so-heavynode' %} - discovery.type=single-node {% endif %} - ES_JAVA_OPTS=-Xms{{ GLOBALS.elasticsearch.es_heap }} -Xmx{{ GLOBALS.elasticsearch.es_heap }} -Des.transport.cname_in_publish_address=true -Dlog4j2.formatMsgNoLookups=true diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings index 9048b85b1..fb1bfa08b 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-cluster-settings @@ -4,7 +4,7 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. {%- from 'vars/globals.map.jinja' import GLOBALS %} -{%- set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} +{%- set node_data = salt['pillar.get']('elasticsearch:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} . /usr/sbin/so-common diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index 4098df21f..5c4b68a76 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -7,23 +7,26 @@ {% import_yaml 'logstash/defaults.yaml' as LOGSTASH_DEFAULTS %} {% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %} -{% set REDIS_NODES = [] %} -{# LOGSTASH_NODES is the same as ES_LOGSTASH_NODES from elasticsearch/config.map.jinja but heavynodes are present #} +{# used to store the redis nodes that logstash needs to know about to pull from the queue #} +{% set LOGSTASH_REDIS_NODES = [] %} +{# stores all logstash nodes #} {% set LOGSTASH_NODES = [] %} -{% set node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} +{% set logstash_node_data = salt['pillar.get']('logstash:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} +{% set redis_node_data = salt['pillar.get']('redis:nodes', {GLOBALS.role.split('-')[1]: {GLOBALS.hostname: {'ip': GLOBALS.node_ip}}}) %} -{% for node_type, node_details in node_data.items() | sort %} +{% for node_type, node_details in redis_node_data.items() | sort %} {% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch', 'so-fleet'] %} {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %} {% for hostname in node_data[node_type].keys() %} -{% do REDIS_NODES.append({hostname:node_details[hostname].ip}) %} +{% do LOGSTASH_REDIS_NODES.append({hostname:node_details[hostname].ip}) %} {% endfor %} {% endif %} -{% else %} -{% do REDIS_NODES.append({GLOBALS.hostname:GLOBALS.node_ip}) %} {% endif %} +{% endfor %} +{% for node_type, node_details in logstash_node_data.items() | sort %} {% for hostname in node_data[node_type].keys() %} {% do LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %} {% endfor %} {% endfor %} + diff --git a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja index 661bc0b73..ad9fae5f2 100644 --- a/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja +++ b/salt/logstash/pipelines/config/so/0900_input_redis.conf.jinja @@ -1,8 +1,8 @@ -{%- from 'logstash/map.jinja' import REDIS_NODES with context %} +{%- from 'logstash/map.jinja' import LOGSTASH_REDIS_NODES with context %} {%- set REDIS_PASS = salt['pillar.get']('redis:config:requirepass') %} -{%- for index in range(REDIS_NODES|length) %} -{%- for host in REDIS_NODES[index] %} +{%- for index in range(LOGSTASH_REDIS_NODES|length) %} +{%- for host in LOGSTASH_REDIS_NODES[index] %} input { redis { host => '{{ host }}' diff --git a/salt/soc/defaults.map.jinja b/salt/soc/defaults.map.jinja index 83cb5637c..b52aa97d8 100644 --- a/salt/soc/defaults.map.jinja +++ b/salt/soc/defaults.map.jinja @@ -14,7 +14,7 @@ {% endfor %} {# add all grid heavy nodes to soc.server.modules.elastic.remoteHostUrls #} -{% for node_type, minions in salt['pillar.get']('logstash:nodes', {}).items() %} +{% for node_type, minions in salt['pillar.get']('elasticsearch:nodes', {}).items() %} {% if node_type in ['heavynode'] %} {% for m in minions.keys() %} {% do SOCDEFAULTS.soc.config.server.modules.elastic.remoteHostUrls.append('https://' ~ m ~ ':9200') %} diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index c823175cb..a3a28d913 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -5,9 +5,8 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %} -{% from 'logstash/map.jinja' import LOGSTASH_NODES %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES as DOCKER_EXTRA_HOSTS %} {% from 'manager/map.jinja' import MANAGERMERGED %} -{% set DOCKER_EXTRA_HOSTS = LOGSTASH_NODES %} {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %} {% set SOCMERGED = salt['pillar.get']('soc', SOCDEFAULTS, merge=true) %} From 469ca4401621c2b651cd12f3c1c9eb8892e46ce4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Thu, 20 Jun 2024 16:53:12 -0400 Subject: [PATCH 07/80] fix maps --- salt/logstash/map.jinja | 4 ++-- salt/soc/merged.map.jinja | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index 5c4b68a76..0bb6de2b7 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -17,7 +17,7 @@ {% for node_type, node_details in redis_node_data.items() | sort %} {% if GLOBALS.role in ['so-searchnode', 'so-standalone', 'so-managersearch', 'so-fleet'] %} {% if node_type in ['manager', 'managersearch', 'standalone', 'receiver' ] %} -{% for hostname in node_data[node_type].keys() %} +{% for hostname in redis_node_data[node_type].keys() %} {% do LOGSTASH_REDIS_NODES.append({hostname:node_details[hostname].ip}) %} {% endfor %} {% endif %} @@ -25,7 +25,7 @@ {% endfor %} {% for node_type, node_details in logstash_node_data.items() | sort %} -{% for hostname in node_data[node_type].keys() %} +{% for hostname in logstash_node_data[node_type].keys() %} {% do LOGSTASH_NODES.append({hostname:node_details[hostname].ip}) %} {% endfor %} {% endfor %} diff --git a/salt/soc/merged.map.jinja b/salt/soc/merged.map.jinja index a3a28d913..38794c903 100644 --- a/salt/soc/merged.map.jinja +++ b/salt/soc/merged.map.jinja @@ -5,8 +5,9 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% from 'soc/defaults.map.jinja' import SOCDEFAULTS with context %} -{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES as DOCKER_EXTRA_HOSTS %} +{% from 'elasticsearch/config.map.jinja' import ELASTICSEARCH_NODES %} {% from 'manager/map.jinja' import MANAGERMERGED %} +{% set DOCKER_EXTRA_HOSTS = ELASTICSEARCH_NODES %} {% do DOCKER_EXTRA_HOSTS.append({GLOBALS.influxdb_host:pillar.node_data[GLOBALS.influxdb_host].ip}) %} {% set SOCMERGED = salt['pillar.get']('soc', SOCDEFAULTS, merge=true) %} From f837ea944a99ba72afa973de627768df0d67c3b5 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 25 Jun 2024 15:39:39 -0400 Subject: [PATCH 08/80] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index d2587d896..38f621b25 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.80 +2.4.90 From dfd5e95c93de98f95b9357e34e3587d25432558b Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 Jun 2024 12:37:28 -0400 Subject: [PATCH 09/80] start soup 2.4.90 --- salt/manager/tools/sbin/soup | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 63df4eb25..656e9b3d9 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -385,6 +385,7 @@ preupgrade_changes() { [[ "$INSTALLEDVERSION" == 2.4.50 ]] && up_to_2.4.60 [[ "$INSTALLEDVERSION" == 2.4.60 ]] && up_to_2.4.70 [[ "$INSTALLEDVERSION" == 2.4.70 ]] && up_to_2.4.80 + [[ "$INSTALLEDVERSION" == 2.4.80 ]] && up_to_2.4.90 true } @@ -403,6 +404,7 @@ postupgrade_changes() { [[ "$POSTVERSION" == 2.4.50 ]] && post_to_2.4.60 [[ "$POSTVERSION" == 2.4.60 ]] && post_to_2.4.70 [[ "$POSTVERSION" == 2.4.70 ]] && post_to_2.4.80 + [[ "$POSTVERSION" == 2.4.80 ]] && post_to_2.4.90 true } @@ -482,6 +484,11 @@ post_to_2.4.80() { POSTVERSION=2.4.80 } +post_to_2.4.90() { + echo "Nothing to apply" + POSTVERSION=2.4.90 +} + repo_sync() { echo "Sync the local repo." su socore -c '/usr/sbin/so-repo-sync' || fail "Unable to complete so-repo-sync." @@ -650,6 +657,11 @@ up_to_2.4.80() { INSTALLEDVERSION=2.4.80 } +up_to_2.4.90() { + echo "Nothing to apply" + INSTALLEDVERSION=2.4.90 +} + add_detection_test_pillars() { if [[ -n "$SOUP_INTERNAL_TESTING" ]]; then echo "Adding detection pillar values for automated testing" From cfcfc6819f72878b20cb61005f90e3f8f9f00594 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 26 Jun 2024 12:53:32 -0400 Subject: [PATCH 10/80] disable logstash in heavynode pillars --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index a7ac0420d..c76fe311e 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -500,7 +500,7 @@ post_to_2.4.80() { } post_to_2.4.90() { - echo "Nothing to apply" + disable_logstash_heavynodes POSTVERSION=2.4.90 } From 0f226cc08ec93b6c56a63cc64f1272de95984124 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 26 Jun 2024 13:59:23 -0400 Subject: [PATCH 11/80] Elastic 8.14.1 --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 05c47a6c1..02d16ad2c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -8,7 +8,7 @@ # Elastic agent is not managed by salt. Because of this we must store this base information in a # script that accompanies the soup system. Since so-common is one of those special soup files, # and since this same logic is required during installation, it's included in this file. -ELASTIC_AGENT_TARBALL_VERSION="8.10.4" +ELASTIC_AGENT_TARBALL_VERSION="8.14.1" ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" From 7d0a961482eb1ebeaf4ce554ff61c7bbbde56aa6 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 26 Jun 2024 14:00:54 -0400 Subject: [PATCH 12/80] Elastic 8.14.1 --- salt/kibana/tools/sbin_jinja/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index 5b4b52560..dd36fd030 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.10.4" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From abdfbba32a29a1154d40dcc642f3c226f5932ed7 Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 26 Jun 2024 14:06:24 -0400 Subject: [PATCH 13/80] Elastic 8.14.1 --- .../integrations/elastic-defend/elastic-defend-endpoints.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index de35f803b..15f08a151 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "8.10.2" + "version": "8.14.0" }, "enabled": true, "policy_id": "endpoints-initial", From 13d4738e8f8ee20a57c15e7420d9c829e602f433 Mon Sep 17 00:00:00 2001 From: Wes Date: Wed, 26 Jun 2024 18:39:53 +0000 Subject: [PATCH 14/80] Elastic 8.14.1 --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index bc503debb..f3370d053 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.10.4","id": "8.10.4","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.1","id": "8.14.1","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From feee80cad91cc0ef20adfe42516c70e972060d31 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 27 Jun 2024 09:01:55 -0400 Subject: [PATCH 15/80] Revert back to 8.10.4 --- .../integrations/elastic-defend/elastic-defend-endpoints.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index 15f08a151..de35f803b 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "8.14.0" + "version": "8.10.2" }, "enabled": true, "policy_id": "endpoints-initial", From 0c5b3f7c1c19568eb4854571fe46afa1edb3f5c1 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 27 Jun 2024 09:03:28 -0400 Subject: [PATCH 16/80] Revert back to 8.10.4 --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index f3370d053..bc503debb 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.1","id": "8.14.1","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.10.4","id": "8.10.4","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From 260d4e44bcd9721a16e0c643da8c00d842955fe7 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 27 Jun 2024 09:04:07 -0400 Subject: [PATCH 17/80] Revert back to 8.10.4 --- salt/kibana/tools/sbin_jinja/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index dd36fd030..5b4b52560 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.10.4" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From 222ebbdec16336901ec131b39b70c7aa756624aa Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 27 Jun 2024 09:05:29 -0400 Subject: [PATCH 18/80] Revert back to 8.10.4 --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 02d16ad2c..05c47a6c1 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -8,7 +8,7 @@ # Elastic agent is not managed by salt. Because of this we must store this base information in a # script that accompanies the soup system. Since so-common is one of those special soup files, # and since this same logic is required during installation, it's included in this file. -ELASTIC_AGENT_TARBALL_VERSION="8.14.1" +ELASTIC_AGENT_TARBALL_VERSION="8.10.4" ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" From 81d874c6ae48e416715bdda6dd0e8b35c5cd24a2 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 27 Jun 2024 10:42:58 -0400 Subject: [PATCH 19/80] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 38f621b25..7d52aac7f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.90 +2.4.0-foxtrot From 5743189eef080d2bb6462aebc674672b86684420 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 27 Jun 2024 10:47:46 -0400 Subject: [PATCH 20/80] Elastic 8.14.1 --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 05c47a6c1..02d16ad2c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -8,7 +8,7 @@ # Elastic agent is not managed by salt. Because of this we must store this base information in a # script that accompanies the soup system. Since so-common is one of those special soup files, # and since this same logic is required during installation, it's included in this file. -ELASTIC_AGENT_TARBALL_VERSION="8.10.4" +ELASTIC_AGENT_TARBALL_VERSION="8.14.1" ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" From 847638442bd181725ee621ab148805e5130eff09 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 27 Jun 2024 10:48:28 -0400 Subject: [PATCH 21/80] Elastic 8.14.1 --- .../integrations/elastic-defend/elastic-defend-endpoints.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index de35f803b..15f08a151 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "8.10.2" + "version": "8.14.0" }, "enabled": true, "policy_id": "endpoints-initial", From 8eb6dcc5b7df7f15bb62ce5d57ae13b695e2cf57 Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 27 Jun 2024 10:49:06 -0400 Subject: [PATCH 22/80] Elastic 8.14.1 --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index bc503debb..f3370d053 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.10.4","id": "8.10.4","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.1","id": "8.14.1","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From 7716f4aff8a4498191c5174c3f13602532e525ad Mon Sep 17 00:00:00 2001 From: weslambert Date: Thu, 27 Jun 2024 10:49:52 -0400 Subject: [PATCH 23/80] Elastic 8.14.1 --- salt/kibana/tools/sbin_jinja/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index 5b4b52560..dd36fd030 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.10.4" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From 92a847e3bd9e62fe6f28d9869afaa7937cae5f46 Mon Sep 17 00:00:00 2001 From: DefensiveDepth Date: Thu, 27 Jun 2024 11:48:54 -0400 Subject: [PATCH 24/80] Fix Fleet setup --- .../tools/sbin_jinja/so-elastic-fleet-setup | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 0748557fd..7e497f6f5 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -53,7 +53,8 @@ fi printf "\n### Create ES Token ###\n" ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq -r .value) -### Create Outputs & Fleet URLs ### +### Create Outputs, Fleet Policy and Fleet URLs ### +# Create the Manager Elasticsearch Output first and set it as the default output printf "\nAdd Manager Elasticsearch Output...\n" ESCACRT=$(openssl x509 -in $INTCA) JSON_STRING=$( jq -n \ @@ -62,7 +63,13 @@ JSON_STRING=$( jq -n \ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" -printf "\nCreate Logstash Output Config if node is not an Import or Eval install\n" +# Create the Manager Fleet Server Host Agent Policy +# This has to be done while the Elasticsearch Output is set to the default Output +printf "Create Manager Fleet Server Policy...\n" +elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120" + +# Now we can create the Logstash Output and set it to to be the default Output +printf "\n\nCreate Logstash Output Config if node is not an Import or Eval install\n" {% if grains.role not in ['so-import', 'so-eval'] %} LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt) LOGSTASHKEY=$(openssl rsa -in /etc/pki/elasticfleet-logstash.key) @@ -101,16 +108,6 @@ printf "\n\n" # Load Elasticsearch templates /usr/sbin/so-elasticsearch-templates-load -# Manager Fleet Server Host -elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120" - -#Temp Fixup for ES Output bug -JSON_STRING=$( jq -n \ - --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \ - '{"name": $NAME,"description": $NAME,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":120,"data_output_id":"so-manager_elasticsearch"}' - ) -curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_{{ GLOBALS.hostname }}" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" - # Initial Endpoints Policy elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600" From 9d725f2b0b610af2c30e231e8199716dd2aeb120 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 28 Jun 2024 13:45:50 -0400 Subject: [PATCH 25/80] fix rule update Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/idstools/tools/sbin_jinja/so-rule-update | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/idstools/tools/sbin_jinja/so-rule-update b/salt/idstools/tools/sbin_jinja/so-rule-update index da4c272dd..4ea79c94e 100755 --- a/salt/idstools/tools/sbin_jinja/so-rule-update +++ b/salt/idstools/tools/sbin_jinja/so-rule-update @@ -11,8 +11,8 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then {%- set proxy = salt['pillar.get']('manager:proxy') %} {%- set noproxy = salt['pillar.get']('manager:no_proxy', '') %} -# Download the rules from the internet {%- if proxy %} +# Download the rules from the internet export http_proxy={{ proxy }} export https_proxy={{ proxy }} export no_proxy="{{ noproxy }}" @@ -20,8 +20,8 @@ if [[ ! "`pidof -x $(basename $0) -o %PPID`" ]]; then mkdir -p /nsm/rules/suricata chown -R socore:socore /nsm/rules/suricata +{%- if not GLOBALS.airgap %} # Download the rules from the internet -{%- if GLOBALS.airgap != 'True' %} {%- if IDSTOOLSMERGED.config.ruleset == 'ETOPEN' %} docker exec so-idstools idstools-rulecat -v --suricata-version 6.0 -o /nsm/rules/suricata/ --merged=/nsm/rules/suricata/emerging-all.rules --force {%- elif IDSTOOLSMERGED.config.ruleset == 'ETPRO' %} From 32d7927a495e79710c23852d108c5e04e5a30dfc Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 1 Jul 2024 15:16:06 +0000 Subject: [PATCH 26/80] Template changes for Elastic 8.14.1 --- salt/elasticsearch/defaults.yaml | 2664 ++++++++++++++++++------------ 1 file changed, 1578 insertions(+), 1086 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 36d673d70..8ca79f01e 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,6 +1,4 @@ elasticsearch: - enabled: false - index_clean: true config: action: destructive_requires_name: true @@ -56,87 +54,8 @@ elasticsearch: enabled: true key: /usr/share/elasticsearch/config/elasticsearch.key verification_mode: none - pipelines: - custom001: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom001 - - pipeline: - name: common - custom002: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom002 - - pipeline: - name: common - custom003: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom003 - - pipeline: - name: common - custom004: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom004 - - pipeline: - name: common - custom005: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom005 - - pipeline: - name: common - custom006: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom006 - - pipeline: - name: common - custom007: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom007 - - pipeline: - name: common - custom008: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom008 - - pipeline: - name: common - custom009: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom009 - - pipeline: - name: common - custom010: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom010 - - pipeline: - name: common + enabled: false + index_clean: true index_settings: global_overrides: index_template: @@ -170,84 +89,13 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-items: - index_sorting: false - index_template: - composed_of: - - so-items-mappings - index_patterns: - - .items-default-** - priority: 500 - template: - mappings: - date_detection: false - settings: - index: - lifecycle: - name: so-items-logs - rollover_alias: ".items-default" - routing: - allocation: - include: - _tier_preference: "data_content" - mapping: - total_fields: - limit: 10000 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc - policy: - phases: - hot: - actions: - rollover: - max_size: 50gb - min_age: 0ms - so-lists: - index_sorting: false - index_template: - composed_of: - - so-lists-mappings - index_patterns: - - .lists-default-** - priority: 500 - template: - mappings: - date_detection: false - settings: - index: - lifecycle: - name: so-lists-logs - rollover_alias: ".lists-default" - routing: - allocation: - include: - _tier_preference: "data_content" - mapping: - total_fields: - limit: 10000 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc - policy: - phases: - hot: - actions: - rollover: - max_size: 50gb - min_age: 0ms so-case: index_sorting: false index_template: composed_of: - case-mappings - case-settings + ignore_missing_component_templates: [] index_patterns: - so-case* priority: 500 @@ -271,142 +119,6 @@ elasticsearch: sort: field: '@timestamp' order: desc - so-detection: - index_sorting: false - index_template: - composed_of: - - detection-mappings - - detection-settings - index_patterns: - - so-detection* - priority: 500 - template: - mappings: - date_detection: false - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - settings: - index: - mapping: - total_fields: - limit: 1500 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc - so-logs-soc: - close: 30 - delete: 365 - index_sorting: false - index_template: - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - container-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - common-settings - - common-dynamic-mappings - data_stream: {} - index_patterns: - - logs-soc-so* - priority: 500 - template: - mappings: - date_detection: false - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - settings: - index: - lifecycle: - name: so-soc-logs - mapping: - total_fields: - limit: 5000 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - warm: 7 so-common: close: 30 delete: 365 @@ -473,6 +185,7 @@ elasticsearch: - common-dynamic-mappings - winlog-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-*-so* priority: 1 @@ -523,6 +236,36 @@ elasticsearch: priority: 50 min_age: 30d warm: 7 + so-detection: + index_sorting: false + index_template: + composed_of: + - detection-mappings + - detection-settings + ignore_missing_component_templates: [] + index_patterns: + - so-detection* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + mapping: + total_fields: + limit: 1500 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc so-endgame: index_sorting: false index_template: @@ -585,6 +328,7 @@ elasticsearch: - common-settings - common-dynamic-mappings - winlog-mappings + ignore_missing_component_templates: [] index_patterns: - endgame* priority: 500 @@ -690,6 +434,7 @@ elasticsearch: - dtc-user_agent-mappings - common-settings - common-dynamic-mappings + ignore_missing_component_templates: [] index_patterns: - so-idh-* priority: 500 @@ -802,6 +547,7 @@ elasticsearch: - common-dynamic-mappings - winlog-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-import-so* priority: 500 @@ -852,6 +598,91 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-items: + index_sorting: false + index_template: + composed_of: + - so-items-mappings + ignore_missing_component_templates: [] + index_patterns: + - .items-default-** + priority: 500 + template: + mappings: + date_detection: false + settings: + index: + lifecycle: + name: so-items-logs + rollover_alias: .items-default + mapping: + total_fields: + limit: 10000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + routing: + allocation: + include: + _tier_preference: data_content + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: + rollover: + max_size: 50gb + min_age: 0ms + so-kismet: + index_sorting: false + index_template: + composed_of: + - kismet-mappings + - source-mappings + - client-mappings + - device-mappings + - network-mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: [] + index_patterns: + - logs-kismet-so* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-kismet-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-kratos: close: 30 delete: 365 @@ -911,6 +742,7 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: [] index_patterns: - logs-kratos-so* priority: 500 @@ -961,6 +793,43 @@ elasticsearch: priority: 50 min_age: 30d warm: 7 + so-lists: + index_sorting: false + index_template: + composed_of: + - so-lists-mappings + ignore_missing_component_templates: [] + index_patterns: + - .lists-default-** + priority: 500 + template: + mappings: + date_detection: false + settings: + index: + lifecycle: + name: so-lists-logs + rollover_alias: .lists-default + mapping: + total_fields: + limit: 10000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + routing: + allocation: + include: + _tier_preference: data_content + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: + rollover: + max_size: 50gb + min_age: 0ms so-logs: index_sorting: false index_template: @@ -973,6 +842,7 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: [] index_patterns: - logs-*-* priority: 225 @@ -1034,6 +904,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-1password.item_usages@custom index_patterns: - logs-1password.item_usages-* priority: 501 @@ -1078,6 +950,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-1password.signin_attempts@custom index_patterns: - logs-1password.signin_attempts-* priority: 501 @@ -1122,6 +996,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-apache.access@custom index_patterns: - logs-apache.access-* priority: 501 @@ -1166,6 +1042,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-apache.error@custom index_patterns: - logs-apache.error-* priority: 501 @@ -1210,6 +1088,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-auditd.log@custom index_patterns: - logs-auditd.log-* priority: 501 @@ -1254,6 +1134,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-auth0.logs@custom index_patterns: - logs-auth0.logs-* priority: 501 @@ -1288,25 +1170,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_cloudfront_logs: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.cloudfront_logs@package + - logs-aws.cloudfront_logs@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.cloudfront_logs@custom index_patterns: - - "logs-aws.cloudfront_logs-*" + - logs-aws.cloudfront_logs-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.cloudfront_logs-logs number_of_replicas: 0 - composed_of: - - "logs-aws.cloudfront_logs@package" - - "logs-aws.cloudfront_logs@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1342,6 +1226,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.cloudtrail@custom index_patterns: - logs-aws.cloudtrail-* priority: 501 @@ -1386,6 +1272,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.cloudwatch_logs@custom index_patterns: - logs-aws.cloudwatch_logs-* priority: 501 @@ -1430,6 +1318,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.ec2_logs@custom index_patterns: - logs-aws.ec2_logs-* priority: 501 @@ -1474,6 +1364,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.elb_logs@custom index_patterns: - logs-aws.elb_logs-* priority: 501 @@ -1518,6 +1410,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.firewall_logs@custom index_patterns: - logs-aws.firewall_logs-* priority: 501 @@ -1552,25 +1446,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_guardduty: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.guardduty@package + - logs-aws.guardduty@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.guardduty@custom index_patterns: - - "logs-aws.guardduty-*" + - logs-aws.guardduty-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.guardduty-logs number_of_replicas: 0 - composed_of: - - "logs-aws.guardduty@package" - - "logs-aws.guardduty@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1596,25 +1492,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_inspector: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.inspector@package + - logs-aws.inspector@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.inspector@custom index_patterns: - - "logs-aws.inspector-*" + - logs-aws.inspector-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.inspector-logs number_of_replicas: 0 - composed_of: - - "logs-aws.inspector@package" - - "logs-aws.inspector@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1650,6 +1548,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.route53_public_logs@custom index_patterns: - logs-aws.route53_public_logs-* priority: 501 @@ -1694,6 +1594,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.route53_resolver_logs@custom index_patterns: - logs-aws.route53_resolver_logs-* priority: 501 @@ -1738,6 +1640,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.s3access@custom index_patterns: - logs-aws.s3access-* priority: 501 @@ -1772,25 +1676,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_securityhub_findings: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.securityhub_findings@package + - logs-aws.securityhub_findings@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.securityhub_findings@custom index_patterns: - - "logs-aws.securityhub_findings-*" + - logs-aws.securityhub_findings-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.securityhub_findings-logs number_of_replicas: 0 - composed_of: - - "logs-aws.securityhub_findings@package" - - "logs-aws.securityhub_findings@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1816,25 +1722,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_securityhub_insights: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-aws.securityhub_insights@package + - logs-aws.securityhub_insights@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-aws.securityhub_insights@custom index_patterns: - - "logs-aws.securityhub_insights-*" + - logs-aws.securityhub_insights-* + priority: 501 template: settings: index: lifecycle: name: so-logs-aws.securityhub_insights-logs number_of_replicas: 0 - composed_of: - - "logs-aws.securityhub_insights@package" - - "logs-aws.securityhub_insights@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -1870,6 +1778,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.vpcflow@custom index_patterns: - logs-aws.vpcflow-* priority: 501 @@ -1914,6 +1824,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-aws.waf@custom index_patterns: - logs-aws.waf-* priority: 501 @@ -1958,6 +1870,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.activitylogs@custom index_patterns: - logs-azure.activitylogs-* priority: 501 @@ -2002,6 +1916,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.application_gateway@custom index_patterns: - logs-azure.application_gateway-* priority: 501 @@ -2046,6 +1962,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.auditlogs@custom index_patterns: - logs-azure.auditlogs-* priority: 501 @@ -2090,6 +2008,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.eventhub@custom index_patterns: - logs-azure.eventhub-* priority: 501 @@ -2134,6 +2054,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.firewall_logs@custom index_patterns: - logs-azure.firewall_logs-* priority: 501 @@ -2178,6 +2100,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.identity_protection@custom index_patterns: - logs-azure.identity_protection-* priority: 501 @@ -2222,6 +2146,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.platformlogs@custom index_patterns: - logs-azure.platformlogs-* priority: 501 @@ -2266,6 +2192,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.provisioning@custom index_patterns: - logs-azure.provisioning-* priority: 501 @@ -2310,6 +2238,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.signinlogs@custom index_patterns: - logs-azure.signinlogs-* priority: 501 @@ -2354,6 +2284,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-azure.springcloudlogs@custom index_patterns: - logs-azure.springcloudlogs-* priority: 501 @@ -2398,6 +2330,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-barracuda.waf@custom index_patterns: - logs-barracuda.waf-* priority: 501 @@ -2442,6 +2376,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-carbonblack_edr.log@custom index_patterns: - logs-carbonblack_edr.log-* priority: 501 @@ -2476,25 +2412,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cef_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-cef.log@package + - logs-cef.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-cef.log@custom index_patterns: - - "logs-cef.log-*" + - logs-cef.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-cef.log-logs number_of_replicas: 0 - composed_of: - - "logs-cef.log@package" - - "logs-cef.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2520,25 +2458,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-checkpoint_x_firewall: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-checkpoint.firewall@package + - logs-checkpoint.firewall@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-checkpoint.firewall@custom index_patterns: - - "logs-checkpoint.firewall-*" + - logs-checkpoint.firewall-* + priority: 501 template: settings: index: lifecycle: name: so-logs-checkpoint.firewall-logs number_of_replicas: 0 - composed_of: - - "logs-checkpoint.firewall@package" - - "logs-checkpoint.firewall@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2574,6 +2514,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_asa.log@custom index_patterns: - logs-cisco_asa.log-* priority: 501 @@ -2618,6 +2560,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.admin@custom index_patterns: - logs-cisco_duo.admin-* priority: 501 @@ -2662,6 +2606,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.auth@custom index_patterns: - logs-cisco_duo.auth-* priority: 501 @@ -2706,6 +2652,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.offline_enrollment@custom index_patterns: - logs-cisco_duo.offline_enrollment-* priority: 501 @@ -2750,6 +2698,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.summary@custom index_patterns: - logs-cisco_duo.summary-* priority: 501 @@ -2794,6 +2744,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_duo.telephony@custom index_patterns: - logs-cisco_duo.telephony-* priority: 501 @@ -2828,25 +2780,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ftd_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-cisco_ftd.log@package + - logs-cisco_ftd.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-cisco_ftd.log@custom index_patterns: - - "logs-cisco_ftd.log-*" + - logs-cisco_ftd.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-cisco_ftd.log-logs number_of_replicas: 0 - composed_of: - - "logs-cisco_ftd.log@package" - - "logs-cisco_ftd.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2872,25 +2826,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ios_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-cisco_ios.log@package + - logs-cisco_ios.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-cisco_ios.log@custom index_patterns: - - "logs-cisco_ios.log-*" + - logs-cisco_ios.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-cisco_ios.log-logs number_of_replicas: 0 - composed_of: - - "logs-cisco_ios.log@package" - - "logs-cisco_ios.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2916,25 +2872,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ise_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-cisco_ise.log@package + - logs-cisco_ise.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-cisco_ise.log@custom index_patterns: - - "logs-cisco_ise.log-*" + - logs-cisco_ise.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-cisco_ise.log-logs number_of_replicas: 0 - composed_of: - - "logs-cisco_ise.log@package" - - "logs-cisco_ise.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -2970,6 +2928,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_meraki.events@custom index_patterns: - logs-cisco_meraki.events-* priority: 501 @@ -3014,6 +2974,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_meraki.log@custom index_patterns: - logs-cisco_meraki.log-* priority: 501 @@ -3058,6 +3020,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cisco_umbrella.log@custom index_patterns: - logs-cisco_umbrella.log-* priority: 501 @@ -3092,25 +3056,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_interface: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.interface@package + - logs-citrix_adc.interface@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.interface@custom index_patterns: - - "logs-citrix_adc.interface-*" + - logs-citrix_adc.interface-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.interface-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.interface@package" - - "logs-citrix_adc.interface@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3136,25 +3102,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_lbvserver: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.lbvserver@package + - logs-citrix_adc.lbvserver@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.lbvserver@custom index_patterns: - - "logs-citrix_adc.lbvserver-*" + - logs-citrix_adc.lbvserver-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.lbvserver-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.lbvserver@package" - - "logs-citrix_adc.lbvserver@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3180,25 +3148,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_service: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.service@package + - logs-citrix_adc.service@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.service@custom index_patterns: - - "logs-citrix_adc.service-*" + - logs-citrix_adc.service-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.service-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.service@package" - - "logs-citrix_adc.service@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3224,25 +3194,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_system: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.system@package + - logs-citrix_adc.system@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.system@custom index_patterns: - - "logs-citrix_adc.system-*" + - logs-citrix_adc.system-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.system-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.system@package" - - "logs-citrix_adc.system@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3268,25 +3240,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_vpn: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_adc.vpn@package + - logs-citrix_adc.vpn@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_adc.vpn@custom index_patterns: - - "logs-citrix_adc.vpn-*" + - logs-citrix_adc.vpn-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_adc.vpn-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_adc.vpn@package" - - "logs-citrix_adc.vpn@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3312,25 +3286,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_waf_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-citrix_waf.log@package + - logs-citrix_waf.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-citrix_waf.log@custom index_patterns: - - "logs-citrix_waf.log-*" + - logs-citrix_waf.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-citrix_waf.log-logs number_of_replicas: 0 - composed_of: - - "logs-citrix_waf.log@package" - - "logs-citrix_waf.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -3366,6 +3342,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cloudflare.audit@custom index_patterns: - logs-cloudflare.audit-* priority: 501 @@ -3410,6 +3388,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-cloudflare.logpull@custom index_patterns: - logs-cloudflare.logpull-* priority: 501 @@ -3454,6 +3434,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-crowdstrike.falcon@custom index_patterns: - logs-crowdstrike.falcon-* priority: 501 @@ -3498,6 +3480,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-crowdstrike.fdr@custom index_patterns: - logs-crowdstrike.fdr-* priority: 501 @@ -3542,6 +3526,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-darktrace.ai_analyst_alert@custom index_patterns: - logs-darktrace.ai_analyst_alert-* priority: 501 @@ -3586,6 +3572,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-darktrace.model_breach_alert@custom index_patterns: - logs-darktrace.model_breach_alert-* priority: 501 @@ -3630,6 +3618,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-darktrace.system_status_alert@custom index_patterns: - logs-darktrace.system_status_alert-* priority: 501 @@ -3675,6 +3665,7 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: [] index_patterns: - logs-detections.alerts-* priority: 501 @@ -3737,6 +3728,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent@custom index_patterns: - logs-elastic_agent-* priority: 501 @@ -3798,6 +3791,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.apm_server@custom index_patterns: - logs-elastic_agent.apm_server-* priority: 501 @@ -3859,6 +3854,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.auditbeat@custom index_patterns: - logs-elastic_agent.auditbeat-* priority: 501 @@ -3917,6 +3914,8 @@ elasticsearch: - logs-elastic_agent.cloudbeat@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 + ignore_missing_component_templates: + - logs-elastic_agent.cloudbeat@custom index_patterns: - logs-elastic_agent.cloudbeat-* priority: 501 @@ -3979,6 +3978,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.endpoint_security@custom index_patterns: - logs-elastic_agent.endpoint_security-* priority: 501 @@ -4035,6 +4036,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.filebeat@custom index_patterns: - logs-elastic_agent.filebeat-* priority: 501 @@ -4091,6 +4094,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.fleet_server@custom index_patterns: - logs-elastic_agent.fleet_server-* priority: 501 @@ -4140,6 +4145,8 @@ elasticsearch: - logs-elastic_agent.heartbeat@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 + ignore_missing_component_templates: + - logs-elastic_agent.heartbeat@custom index_patterns: - logs-elastic_agent.heartbeat-* priority: 501 @@ -4202,6 +4209,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.metricbeat@custom index_patterns: - logs-elastic_agent.metricbeat-* priority: 501 @@ -4258,6 +4267,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.osquerybeat@custom index_patterns: - logs-elastic_agent.osquerybeat-* priority: 501 @@ -4313,6 +4324,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-elastic_agent.packetbeat@custom index_patterns: - logs-elastic_agent.packetbeat-* priority: 501 @@ -4375,6 +4388,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.alerts@custom index_patterns: - logs-endpoint.alerts-* priority: 501 @@ -4431,6 +4446,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.diagnostic.collection@custom index_patterns: - .logs-endpoint.diagnostic.collection-* priority: 501 @@ -4487,6 +4504,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.api@custom index_patterns: - logs-endpoint.events.api-* priority: 501 @@ -4543,6 +4562,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.file@custom index_patterns: - logs-endpoint.events.file-* priority: 501 @@ -4599,6 +4620,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.library@custom index_patterns: - logs-endpoint.events.library-* priority: 501 @@ -4655,6 +4678,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.network@custom index_patterns: - logs-endpoint.events.network-* priority: 501 @@ -4711,6 +4736,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.process@custom index_patterns: - logs-endpoint.events.process-* priority: 501 @@ -4767,6 +4794,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.registry@custom index_patterns: - logs-endpoint.events.registry-* priority: 501 @@ -4823,6 +4852,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-endpoint.events.security@custom index_patterns: - logs-endpoint.events.security-* priority: 501 @@ -4878,6 +4909,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-f5_bigip.log@custom index_patterns: - logs-f5_bigip.log-* priority: 501 @@ -4922,6 +4955,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fim.event@custom index_patterns: - logs-fim.event-* priority: 501 @@ -4966,6 +5001,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fireeye.nx@custom index_patterns: - logs-fireeye.nx-* priority: 501 @@ -5010,6 +5047,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet_fortigate.log@custom index_patterns: - logs-fortinet_fortigate.log-* priority: 501 @@ -5054,6 +5093,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet.clientendpoint@custom index_patterns: - logs-fortinet.clientendpoint-* priority: 501 @@ -5098,6 +5139,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet.firewall@custom index_patterns: - logs-fortinet.firewall-* priority: 501 @@ -5142,6 +5185,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet.fortimail@custom index_patterns: - logs-fortinet.fortimail-* priority: 501 @@ -5186,6 +5231,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-fortinet.fortimanager@custom index_patterns: - logs-fortinet.fortimanager-* priority: 501 @@ -5230,6 +5277,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.audit@custom index_patterns: - logs-gcp.audit-* priority: 501 @@ -5274,6 +5323,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.dns@custom index_patterns: - logs-gcp.dns-* priority: 501 @@ -5318,6 +5369,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.firewall@custom index_patterns: - logs-gcp.firewall-* priority: 501 @@ -5362,6 +5415,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.loadbalancing_logs@custom index_patterns: - logs-gcp.loadbalancing_logs-* priority: 501 @@ -5406,6 +5461,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-gcp.vpcflow@custom index_patterns: - logs-gcp.vpcflow-* priority: 501 @@ -5450,6 +5507,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.audit@custom index_patterns: - logs-github.audit-* priority: 501 @@ -5494,6 +5553,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.code_scanning@custom index_patterns: - logs-github.code_scanning-* priority: 501 @@ -5538,6 +5599,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.dependabot@custom index_patterns: - logs-github.dependabot-* priority: 501 @@ -5582,6 +5645,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.issues@custom index_patterns: - logs-github.issues-* priority: 501 @@ -5626,6 +5691,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-github.secret_scanning@custom index_patterns: - logs-github.secret_scanning-* priority: 501 @@ -5670,6 +5737,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.access_transparency@custom index_patterns: - logs-google_workspace.access_transparency-* priority: 501 @@ -5714,6 +5783,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.admin@custom index_patterns: - logs-google_workspace.admin-* priority: 501 @@ -5758,6 +5829,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.alert@custom index_patterns: - logs-google_workspace.alert-* priority: 501 @@ -5802,6 +5875,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.context_aware_access@custom index_patterns: - logs-google_workspace.context_aware_access-* priority: 501 @@ -5846,6 +5921,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.device@custom index_patterns: - logs-google_workspace.device-* priority: 501 @@ -5890,6 +5967,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.drive@custom index_patterns: - logs-google_workspace.drive-* priority: 501 @@ -5934,6 +6013,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.gcp@custom index_patterns: - logs-google_workspace.gcp-* priority: 501 @@ -5978,6 +6059,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.group_enterprise@custom index_patterns: - logs-google_workspace.group_enterprise-* priority: 501 @@ -6022,6 +6105,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.groups@custom index_patterns: - logs-google_workspace.groups-* priority: 501 @@ -6066,6 +6151,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.login@custom index_patterns: - logs-google_workspace.login-* priority: 501 @@ -6110,6 +6197,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.rules@custom index_patterns: - logs-google_workspace.rules-* priority: 501 @@ -6154,6 +6243,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.saml@custom index_patterns: - logs-google_workspace.saml-* priority: 501 @@ -6198,6 +6289,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.token@custom index_patterns: - logs-google_workspace.token-* priority: 501 @@ -6242,6 +6335,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-google_workspace.user_accounts@custom index_patterns: - logs-google_workspace.user_accounts-* priority: 501 @@ -6286,6 +6381,9 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-http_endpoint.generic@package + - logs-http_endpoint.generic@custom index_patterns: - logs-http_endpoint.generic-* priority: 501 @@ -6330,6 +6428,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-httpjson.generic@custom index_patterns: - logs-httpjson.generic-* priority: 501 @@ -6364,25 +6464,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-iis_x_access: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-iis.access@package + - logs-iis.access@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-iis.access@custom index_patterns: - - "logs-iis.access-*" + - logs-iis.access-* + priority: 501 template: settings: index: lifecycle: name: so-logs-iis.access-logs number_of_replicas: 0 - composed_of: - - "logs-iis.access@package" - - "logs-iis.access@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -6408,25 +6510,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-iis_x_error: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-iis.error@package + - logs-iis.error@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-iis.error@custom index_patterns: - - "logs-iis.error-*" + - logs-iis.error-* + priority: 501 template: settings: index: lifecycle: name: so-logs-iis.error-logs number_of_replicas: 0 - composed_of: - - "logs-iis.error@package" - - "logs-iis.error@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -6462,6 +6566,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-juniper_srx.log@custom index_patterns: - logs-juniper_srx.log-* priority: 501 @@ -6506,6 +6612,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-juniper.junos@custom index_patterns: - logs-juniper.junos-* priority: 501 @@ -6550,6 +6658,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-juniper.netscreen@custom index_patterns: - logs-juniper.netscreen-* priority: 501 @@ -6594,6 +6704,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-juniper.srx@custom index_patterns: - logs-juniper.srx-* priority: 501 @@ -6638,6 +6750,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-kafka_log.generic@custom index_patterns: - logs-kafka_log.generic-* priority: 501 @@ -6682,6 +6796,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-lastpass.detailed_shared_folder@custom index_patterns: - logs-lastpass.detailed_shared_folder-* priority: 501 @@ -6726,6 +6842,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-lastpass.event_report@custom index_patterns: - logs-lastpass.event_report-* priority: 501 @@ -6770,6 +6888,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-lastpass.user@custom index_patterns: - logs-lastpass.user-* priority: 501 @@ -6814,6 +6934,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-m365_defender.event@custom index_patterns: - logs-m365_defender.event-* priority: 501 @@ -6858,6 +6980,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-m365_defender.incident@custom index_patterns: - logs-m365_defender.incident-* priority: 501 @@ -6902,6 +7026,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-m365_defender.log@custom index_patterns: - logs-m365_defender.log-* priority: 501 @@ -6946,6 +7072,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-microsoft_defender_endpoint.log@custom index_patterns: - logs-microsoft_defender_endpoint.log-* priority: 501 @@ -6990,6 +7118,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-microsoft_dhcp.log@custom index_patterns: - logs-microsoft_dhcp.log-* priority: 501 @@ -7024,25 +7154,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-microsoft_sqlserver_x_audit: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-microsoft_sqlserver.audit@package + - logs-microsoft_sqlserver.audit@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-microsoft_sqlserver.audit@custom index_patterns: - - "logs-microsoft_sqlserver.audit-*" + - logs-microsoft_sqlserver.audit-* + priority: 501 template: settings: index: lifecycle: name: so-logs-microsoft_sqlserver.audit-logs number_of_replicas: 0 - composed_of: - - "logs-microsoft_sqlserver.audit@package" - - "logs-microsoft_sqlserver.audit@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7068,113 +7200,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-microsoft_sqlserver_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-microsoft_sqlserver.log@package + - logs-microsoft_sqlserver.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-microsoft_sqlserver.log@custom index_patterns: - - "logs-microsoft_sqlserver.log-*" + - logs-microsoft_sqlserver.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-microsoft_sqlserver.log-logs number_of_replicas: 0 - composed_of: - - "logs-microsoft_sqlserver.log@package" - - "logs-microsoft_sqlserver.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-logs-mysql_x_error: - index_sorting: False - index_template: - index_patterns: - - "logs-mysql.error-*" - template: - settings: - index: - lifecycle: - name: so-logs-mysql.error-logs - number_of_replicas: 0 - composed_of: - - "logs-mysql.error@package" - - "logs-mysql.error@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-logs-mysql_x_slowlog: - index_sorting: False - index_template: - index_patterns: - - "logs-mysql.slowlog-*" - template: - settings: - index: - lifecycle: - name: so-logs-mysql.slowlog-logs - number_of_replicas: 0 - composed_of: - - "logs-mysql.slowlog@package" - - "logs-mysql.slowlog@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7210,6 +7256,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.audit_events@custom index_patterns: - logs-mimecast.audit_events-* priority: 501 @@ -7254,6 +7302,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.dlp_logs@custom index_patterns: - logs-mimecast.dlp_logs-* priority: 501 @@ -7298,6 +7348,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.siem_logs@custom index_patterns: - logs-mimecast.siem_logs-* priority: 501 @@ -7342,6 +7394,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.threat_intel_malware_customer@custom index_patterns: - logs-mimecast.threat_intel_malware_customer-* priority: 501 @@ -7386,6 +7440,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.threat_intel_malware_grid@custom index_patterns: - logs-mimecast.threat_intel_malware_grid-* priority: 501 @@ -7430,6 +7486,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.ttp_ap_logs@custom index_patterns: - logs-mimecast.ttp_ap_logs-* priority: 501 @@ -7474,6 +7532,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.ttp_ip_logs@custom index_patterns: - logs-mimecast.ttp_ip_logs-* priority: 501 @@ -7518,6 +7578,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-mimecast.ttp_url_logs@custom index_patterns: - logs-mimecast.ttp_url_logs-* priority: 501 @@ -7551,6 +7613,98 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-mysql_x_error: + index_sorting: false + index_template: + composed_of: + - logs-mysql.error@package + - logs-mysql.error@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-mysql.error@custom + index_patterns: + - logs-mysql.error-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-mysql.error-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-mysql_x_slowlog: + index_sorting: false + index_template: + composed_of: + - logs-mysql.slowlog@package + - logs-mysql.slowlog@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-mysql.slowlog@custom + index_patterns: + - logs-mysql.slowlog-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-logs-mysql.slowlog-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logs-netflow_x_log: index_sorting: false index_template: @@ -7562,6 +7716,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-netflow.log@custom index_patterns: - logs-netflow.log-* priority: 501 @@ -7596,25 +7752,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-nginx_x_access: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-nginx.access@package + - logs-nginx.access@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-nginx.access@custom index_patterns: - - "logs-nginx.access-*" + - logs-nginx.access-* + priority: 501 template: settings: index: lifecycle: name: so-logs-nginx.access-logs number_of_replicas: 0 - composed_of: - - "logs-nginx.access@package" - - "logs-nginx.access@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7640,69 +7798,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-nginx_x_error: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-nginx.error@package + - logs-nginx.error@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-nginx.error@custom index_patterns: - - "logs-nginx.error-*" + - logs-nginx.error-* + priority: 501 template: settings: index: lifecycle: name: so-logs-nginx.error-logs number_of_replicas: 0 - composed_of: - - "logs-nginx.error@package" - - "logs-nginx.error@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-nginx_x_stubstatus: - index_sorting: False - index_template: - index_patterns: - - "metrics-nginx.stubstatus-*" - template: - settings: - index: - lifecycle: - name: so-metrics-nginx.stubstatus-logs - number_of_replicas: 0 - composed_of: - - "metrics-nginx.stubstatus@package" - - "metrics-nginx.stubstatus@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7738,6 +7854,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-o365.audit@custom index_patterns: - logs-o365.audit-* priority: 501 @@ -7782,6 +7900,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-okta.system@custom index_patterns: - logs-okta.system-* priority: 501 @@ -7825,6 +7945,7 @@ elasticsearch: name: elastic_agent composed_of: - logs-osquery_manager.action.responses + ignore_missing_component_templates: [] index_patterns: - .logs-osquery_manager.action.responses* priority: 501 @@ -7842,6 +7963,7 @@ elasticsearch: name: elastic_agent composed_of: - logs-osquery_manager.actions + ignore_missing_component_templates: [] index_patterns: - .logs-osquery_manager.actions* priority: 501 @@ -7860,6 +7982,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-panw.panos@custom index_patterns: - logs-panw.panos-* priority: 501 @@ -7904,6 +8028,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-pfsense.log@custom index_patterns: - logs-pfsense.log-* priority: 501 @@ -7938,25 +8064,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_clicks_blocked: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-proofpoint_tap.clicks_blocked@package + - logs-proofpoint_tap.clicks_blocked@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-proofpoint_tap.clicks_blocked@custom index_patterns: - - "logs-proofpoint_tap.clicks_blocked-*" + - logs-proofpoint_tap.clicks_blocked-* + priority: 501 template: settings: index: lifecycle: name: so-logs-proofpoint_tap.clicks_blocked-logs number_of_replicas: 0 - composed_of: - - "logs-proofpoint_tap.clicks_blocked@package" - - "logs-proofpoint_tap.clicks_blocked@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -7982,25 +8110,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_clicks_permitted: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-proofpoint_tap.clicks_permitted@package + - logs-proofpoint_tap.clicks_permitted@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-proofpoint_tap.clicks_permitted@custom index_patterns: - - "logs-proofpoint_tap.clicks_permitted-*" + - logs-proofpoint_tap.clicks_permitted-* + priority: 501 template: settings: index: lifecycle: name: so-logs-proofpoint_tap.clicks_permitted-logs number_of_replicas: 0 - composed_of: - - "logs-proofpoint_tap.clicks_permitted@package" - - "logs-proofpoint_tap.clicks_permitted@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -8026,25 +8156,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_message_blocked: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-proofpoint_tap.message_blocked@package + - logs-proofpoint_tap.message_blocked@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-proofpoint_tap.message_blocked@custom index_patterns: - - "logs-proofpoint_tap.message_blocked-*" + - logs-proofpoint_tap.message_blocked-* + priority: 501 template: settings: index: lifecycle: name: so-logs-proofpoint_tap.message_blocked-logs number_of_replicas: 0 - composed_of: - - "logs-proofpoint_tap.message_blocked@package" - - "logs-proofpoint_tap.message_blocked@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -8070,25 +8202,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_message_delivered: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-proofpoint_tap.message_delivered@package + - logs-proofpoint_tap.message_delivered@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-proofpoint_tap.message_delivered@custom index_patterns: - - "logs-proofpoint_tap.message_delivered-*" + - logs-proofpoint_tap.message_delivered-* + priority: 501 template: settings: index: lifecycle: name: so-logs-proofpoint_tap.message_delivered-logs number_of_replicas: 0 - composed_of: - - "logs-proofpoint_tap.message_delivered@package" - - "logs-proofpoint_tap.message_delivered@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -8124,6 +8258,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-pulse_connect_secure.log@custom index_patterns: - logs-pulse_connect_secure.log-* priority: 501 @@ -8168,6 +8304,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.activity@custom index_patterns: - logs-sentinel_one.activity-* priority: 501 @@ -8212,6 +8350,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.agent@custom index_patterns: - logs-sentinel_one.agent-* priority: 501 @@ -8256,6 +8396,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.alert@custom index_patterns: - logs-sentinel_one.alert-* priority: 501 @@ -8300,6 +8442,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.group@custom index_patterns: - logs-sentinel_one.group-* priority: 501 @@ -8344,6 +8488,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sentinel_one.threat@custom index_patterns: - logs-sentinel_one.threat-* priority: 501 @@ -8378,25 +8524,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-snort_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-snort.log@package + - logs-snort.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-snort.log@custom index_patterns: - - "logs-snort.log-*" + - logs-snort.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-snort.log-logs number_of_replicas: 0 - composed_of: - - "logs-snort.log@package" - - "logs-snort.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -8432,6 +8580,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-snyk.audit@custom index_patterns: - logs-snyk.audit-* priority: 501 @@ -8476,6 +8626,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-snyk.vulnerabilities@custom index_patterns: - logs-snyk.vulnerabilities-* priority: 501 @@ -8509,6 +8661,114 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-logs-soc: + close: 30 + delete: 365 + index_sorting: false + index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - container-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - threat-mappings + - tls-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - common-settings + - common-dynamic-mappings + data_stream: {} + ignore_missing_component_templates: [] + index_patterns: + - logs-soc-so* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + lifecycle: + name: so-soc-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + warm: 7 so-logs-sonicwall_firewall_x_log: index_sorting: false index_template: @@ -8520,6 +8780,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sonicwall_firewall.log@custom index_patterns: - logs-sonicwall_firewall.log-* priority: 501 @@ -8564,6 +8826,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sophos_central.alert@custom index_patterns: - logs-sophos_central.alert-* priority: 501 @@ -8608,6 +8872,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sophos_central.event@custom index_patterns: - logs-sophos_central.event-* priority: 501 @@ -8652,6 +8918,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sophos.utm@custom index_patterns: - logs-sophos.utm-* priority: 501 @@ -8696,6 +8964,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-sophos.xg@custom index_patterns: - logs-sophos.xg-* priority: 501 @@ -8740,6 +9010,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-symantec_endpoint.log@custom index_patterns: - logs-symantec_endpoint.log-* priority: 501 @@ -8785,6 +9057,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.application@custom index_patterns: - logs-system.application* priority: 501 @@ -8830,6 +9104,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.auth@custom index_patterns: - logs-system.auth* priority: 501 @@ -8875,6 +9151,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.security@custom index_patterns: - logs-system.security* priority: 501 @@ -8920,6 +9198,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.syslog@custom index_patterns: - logs-system.syslog* priority: 501 @@ -8965,6 +9245,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-system.system@custom index_patterns: - logs-system.system* priority: 501 @@ -9009,6 +9291,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-tenable_sc.asset@custom index_patterns: - logs-tenable_sc.asset-* priority: 501 @@ -9053,6 +9337,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-tenable_sc.plugin@custom index_patterns: - logs-tenable_sc.plugin-* priority: 501 @@ -9097,6 +9383,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-tenable_sc.vulnerability@custom index_patterns: - logs-tenable_sc.vulnerability-* priority: 501 @@ -9141,6 +9429,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_abusech.malware@custom index_patterns: - logs-ti_abusech.malware-* priority: 501 @@ -9185,6 +9475,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_abusech.malwarebazaar@custom index_patterns: - logs-ti_abusech.malwarebazaar-* priority: 501 @@ -9229,6 +9521,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_abusech.threatfox@custom index_patterns: - logs-ti_abusech.threatfox-* priority: 501 @@ -9273,6 +9567,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_abusech.url@custom index_patterns: - logs-ti_abusech.url-* priority: 501 @@ -9307,25 +9603,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_anomali_x_threatstream: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-ti_anomali.threatstream@package + - logs-ti_anomali.threatstream@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-ti_anomali.threatstream@custom index_patterns: - - "logs-ti_anomali.threatstream-*" + - logs-ti_anomali.threatstream-* + priority: 501 template: settings: index: lifecycle: name: so-logs-ti_anomali.threatstream-logs number_of_replicas: 0 - composed_of: - - "logs-ti_anomali.threatstream@package" - - "logs-ti_anomali.threatstream@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9351,25 +9649,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_cybersixgill_x_threat: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-ti_cybersixgill.threat@package + - logs-ti_cybersixgill.threat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-ti_cybersixgill.threat@custom index_patterns: - - "logs-ti_cybersixgill.threat-*" + - logs-ti_cybersixgill.threat-* + priority: 501 template: settings: index: lifecycle: name: so-logs-ti_cybersixgill.threat-logs number_of_replicas: 0 - composed_of: - - "logs-ti_cybersixgill.threat@package" - - "logs-ti_cybersixgill.threat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9405,6 +9705,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_misp.threat@custom index_patterns: - logs-ti_misp.threat-* priority: 501 @@ -9449,6 +9751,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_misp.threat_attributes@custom index_patterns: - logs-ti_misp.threat_attributes-* priority: 501 @@ -9493,6 +9797,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_otx.pulses_subscribed@custom index_patterns: - logs-ti_otx.pulses_subscribed-* priority: 501 @@ -9537,6 +9843,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_otx.threat@custom index_patterns: - logs-ti_otx.threat-* priority: 501 @@ -9581,6 +9889,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_recordedfuture.latest_ioc-template@custom index_patterns: - logs-ti_recordedfuture.latest_ioc-template-* priority: 501 @@ -9625,6 +9935,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-ti_recordedfuture.threat@custom index_patterns: - logs-ti_recordedfuture.threat-* priority: 501 @@ -9659,25 +9971,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_threatq_x_threat: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-ti_threatq.threat@package + - logs-ti_threatq.threat@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-ti_threatq.threat@custom index_patterns: - - "logs-ti_threatq.threat-*" + - logs-ti_threatq.threat-* + priority: 501 template: settings: index: lifecycle: name: so-logs-ti_threatq.threat-logs number_of_replicas: 0 - composed_of: - - "logs-ti_threatq.threat@package" - - "logs-ti_threatq.threat@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9703,25 +10017,27 @@ elasticsearch: priority: 50 min_age: 30d so-logs-vsphere_x_log: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-vsphere.log@package + - logs-vsphere.log@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-vsphere.log@custom index_patterns: - - "logs-vsphere.log-*" + - logs-vsphere.log-* + priority: 501 template: settings: index: lifecycle: name: so-logs-vsphere.log-logs number_of_replicas: 0 - composed_of: - - "logs-vsphere.log@package" - - "logs-vsphere.log@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9757,6 +10073,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-windows.forwarded@custom index_patterns: - logs-windows.forwarded* priority: 501 @@ -9801,6 +10119,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-windows.powershell@custom index_patterns: - logs-windows.powershell-* priority: 501 @@ -9845,6 +10165,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-windows.powershell_operational@custom index_patterns: - logs-windows.powershell_operational-* priority: 501 @@ -9889,6 +10211,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-windows.sysmon_operational@custom index_patterns: - logs-windows.sysmon_operational-* priority: 501 @@ -9923,25 +10247,28 @@ elasticsearch: priority: 50 min_age: 30d so-logs-winlog_x_winlog: - index_sorting: False + index_sorting: false index_template: + composed_of: + - logs-winlog.winlog@package + - logs-winlog.winlog@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - logs-winlog.winlog@package + - logs-winlog.winlog@custom index_patterns: - - "logs-winlog.winlog-*" + - logs-winlog.winlog-* + priority: 501 template: settings: index: lifecycle: name: so-logs-winlog.winlog-logs number_of_replicas: 0 - composed_of: - - "logs-winlog.winlog@package" - - "logs-winlog.winlog@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false policy: phases: cold: @@ -9977,6 +10304,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.alerts@custom index_patterns: - logs-zscaler_zia.alerts-* priority: 501 @@ -10021,6 +10350,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.dns@custom index_patterns: - logs-zscaler_zia.dns-* priority: 501 @@ -10065,6 +10396,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.firewall@custom index_patterns: - logs-zscaler_zia.firewall-* priority: 501 @@ -10109,6 +10442,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.tunnel@custom index_patterns: - logs-zscaler_zia.tunnel-* priority: 501 @@ -10153,6 +10488,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zia.web@custom index_patterns: - logs-zscaler_zia.web-* priority: 501 @@ -10197,6 +10534,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.app_connector_status@custom index_patterns: - logs-zscaler_zpa.app_connector_status-* priority: 501 @@ -10241,6 +10580,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.audit@custom index_patterns: - logs-zscaler_zpa.audit-* priority: 501 @@ -10285,6 +10626,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.browser_access@custom index_patterns: - logs-zscaler_zpa.browser_access-* priority: 501 @@ -10329,6 +10672,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.user_activity@custom index_patterns: - logs-zscaler_zpa.user_activity-* priority: 501 @@ -10373,6 +10718,8 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false + ignore_missing_component_templates: + - logs-zscaler_zpa.user_status@custom index_patterns: - logs-zscaler_zpa.user_status-* priority: 501 @@ -10406,317 +10753,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-metrics-endpoint_x_metadata: - index_sorting: False - index_template: - index_patterns: - - "metrics-endpoint.metadata-*" - template: - settings: - index: - lifecycle: - name: so-metrics-endpoint.metadata-logs - number_of_replicas: 0 - composed_of: - - "metrics-endpoint.metadata@package" - - "metrics-endpoint.metadata@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-endpoint_x_metrics: - index_sorting: False - index_template: - index_patterns: - - "metrics-endpoint.metrics-*" - template: - settings: - index: - lifecycle: - name: so-metrics-endpoint.metrics-logs - number_of_replicas: 0 - composed_of: - - "metrics-endpoint.metrics@package" - - "metrics-endpoint.metrics@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-endpoint_x_policy: - index_sorting: False - index_template: - index_patterns: - - "metrics-endpoint.policy-*" - template: - settings: - index: - lifecycle: - name: so-metrics-endpoint.policy-logs - number_of_replicas: 0 - composed_of: - - "metrics-endpoint.policy@package" - - "metrics-endpoint.policy@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_datastore: - index_sorting: False - index_template: - index_patterns: - - "metrics-vsphere.datastore-*" - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.datastore-logs - number_of_replicas: 0 - composed_of: - - "metrics-vsphere.datastore@package" - - "metrics-vsphere.datastore@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_host: - index_sorting: False - index_template: - index_patterns: - - "metrics-vsphere.host-*" - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.host-logs - number_of_replicas: 0 - composed_of: - - "metrics-vsphere.host@package" - - "metrics-vsphere.host@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_virtualmachine: - index_sorting: False - index_template: - index_patterns: - - "metrics-vsphere.virtualmachine-*" - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.virtualmachine-logs - number_of_replicas: 0 - composed_of: - - "metrics-vsphere.virtualmachine@package" - - "metrics-vsphere.virtualmachine@custom" - - "so-fleet_globals-1" - - "so-fleet_agent_id_verification-1" - priority: 501 - data_stream: - hidden: false - allow_custom_routing: false - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-kismet: - index_sorting: false - index_template: - composed_of: - - kismet-mappings - - source-mappings - - client-mappings - - device-mappings - - network-mappings - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - index_patterns: - - logs-kismet-so* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-kismet-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d so-logstash: index_sorting: false index_template: @@ -10778,6 +10814,7 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings + ignore_missing_component_templates: [] index_patterns: - logs-logstash-default* priority: 500 @@ -10827,6 +10864,374 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-metrics-endpoint_x_metadata: + index_sorting: false + index_template: + composed_of: + - metrics-endpoint.metadata@package + - metrics-endpoint.metadata@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-endpoint.metadata@custom + index_patterns: + - metrics-endpoint.metadata-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-endpoint.metadata-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-endpoint_x_metrics: + index_sorting: false + index_template: + composed_of: + - metrics-endpoint.metrics@package + - metrics-endpoint.metrics@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-endpoint.metrics@custom + index_patterns: + - metrics-endpoint.metrics-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-endpoint.metrics-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-endpoint_x_policy: + index_sorting: false + index_template: + composed_of: + - metrics-endpoint.policy@package + - metrics-endpoint.policy@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-endpoint.policy@custom + index_patterns: + - metrics-endpoint.policy-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-endpoint.policy-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-fleet_server_x_agent_status: + index_sorting: false + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-fleet_server.agent_status@package + - metrics-fleet_server.agent_status@custom + - ecs@mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-fleet_server.agent_status@custom + index_patterns: + - metrics-fleet_server.agent_status-* + priority: 501 + template: + settings: + index: + mode: time_series + number_of_replicas: 0 + so-metrics-fleet_server_x_agent_versions: + index_sorting: false + index_template: + composed_of: + - metrics@tsdb-settings + - metrics-fleet_server.agent_versions@package + - metrics-fleet_server.agent_versions@custom + - ecs@mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-fleet_server.agent_versions@custom + index_patterns: + - metrics-fleet_server.agent_versions-* + priority: 501 + template: + settings: + index: + mode: time_series + number_of_replicas: 0 + so-metrics-nginx_x_stubstatus: + index_sorting: false + index_template: + composed_of: + - metrics-nginx.stubstatus@package + - metrics-nginx.stubstatus@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-nginx.stubstatus@custom + index_patterns: + - metrics-nginx.stubstatus-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-nginx.stubstatus-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-vsphere_x_datastore: + index_sorting: false + index_template: + composed_of: + - metrics-vsphere.datastore@package + - metrics-vsphere.datastore@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-vsphere.datastore@custom + index_patterns: + - metrics-vsphere.datastore-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.datastore-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-vsphere_x_host: + index_sorting: false + index_template: + composed_of: + - metrics-vsphere.host@package + - metrics-vsphere.host@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-vsphere.host@custom + index_patterns: + - metrics-vsphere.host-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.host-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-vsphere_x_virtualmachine: + index_sorting: false + index_template: + composed_of: + - metrics-vsphere.virtualmachine@package + - metrics-vsphere.virtualmachine@custom + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + ignore_missing_component_templates: + - metrics-vsphere.virtualmachine@custom + index_patterns: + - metrics-vsphere.virtualmachine-* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.virtualmachine-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-redis: index_sorting: false index_template: @@ -10888,6 +11293,7 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings + ignore_missing_component_templates: [] index_patterns: - logs-redis-default* priority: 500 @@ -11000,6 +11406,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-strelka-so* priority: 500 @@ -11111,6 +11518,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-suricata-so* priority: 500 @@ -11222,6 +11630,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-suricata.alerts-* priority: 500 @@ -11334,6 +11743,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-syslog-so* priority: 500 @@ -11447,6 +11857,7 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} + ignore_missing_component_templates: [] index_patterns: - logs-zeek-so* priority: 500 @@ -11496,6 +11907,87 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + pipelines: + custom001: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom001 + - pipeline: + name: common + custom002: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom002 + - pipeline: + name: common + custom003: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom003 + - pipeline: + name: common + custom004: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom004 + - pipeline: + name: common + custom005: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom005 + - pipeline: + name: common + custom006: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom006 + - pipeline: + name: common + custom007: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom007 + - pipeline: + name: common + custom008: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom008 + - pipeline: + name: common + custom009: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom009 + - pipeline: + name: common + custom010: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom010 + - pipeline: + name: common retention: retention_pct: 50 so_roles: From 1b47d5c62286e2fe3276e9e4bf943254c61c403a Mon Sep 17 00:00:00 2001 From: Wes Date: Mon, 1 Jul 2024 15:16:58 +0000 Subject: [PATCH 27/80] Changes for Elastic 8.14.1 --- .../tools/sbin_jinja/so-elasticsearch-templates-load | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 080348522..12ef4dbf6 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -134,7 +134,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then TEMPLATE=${i::-14} COMPONENT_PATTERN=${TEMPLATE:3} MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery") - if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" ]]; then + if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ logs-http_endpoint\.generic|logs-winlog\.winlog ]]; then load_failures=$((load_failures+1)) echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures" else @@ -153,7 +153,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then cd - >/dev/null if [[ $load_failures -eq 0 ]]; then - echo "All template loaded successfully" + echo "All templates loaded successfully" touch $STATE_FILE_SUCCESS else echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate" From 4d499be1a83dd3695d6e7c47a19f1b2b5394d40e Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 2 Jul 2024 08:47:29 -0400 Subject: [PATCH 28/80] Change name --- .../files/integrations/endpoints-initial/windows-defender.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json index ac4394e62..ab7e0783f 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json @@ -11,7 +11,7 @@ "winlogs-winlog": { "enabled": true, "streams": { - "winlog.winlog": { + "winlog.winlogs": { "enabled": true, "vars": { "channel": "Microsoft-Windows-Windows Defender/Operational", From 745b6775f1ec2d5c77192dee7b0f1c72155ea4f2 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 2 Jul 2024 09:05:35 -0400 Subject: [PATCH 29/80] Change name for ILM --- salt/elasticsearch/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 8ca79f01e..98eac89f2 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -8734,7 +8734,7 @@ elasticsearch: settings: index: lifecycle: - name: so-soc-logs + name: so-logs-soc-logs mapping: total_fields: limit: 5000 From 578c6c567f84c88fccc50c151e3980b406c4cd3d Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 2 Jul 2024 14:34:45 -0400 Subject: [PATCH 30/80] Delete old user commands --- salt/manager/tools/sbin/so-user-add | 2 -- 1 file changed, 2 deletions(-) delete mode 100755 salt/manager/tools/sbin/so-user-add diff --git a/salt/manager/tools/sbin/so-user-add b/salt/manager/tools/sbin/so-user-add deleted file mode 100755 index 2685365b6..000000000 --- a/salt/manager/tools/sbin/so-user-add +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -so-user add --email $1 \ No newline at end of file From 8dc8092241ff5c274a76ee7773dada8e20704bce Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 2 Jul 2024 14:36:02 -0400 Subject: [PATCH 31/80] Delete salt/manager/tools/sbin/so-user-disable --- salt/manager/tools/sbin/so-user-disable | 2 -- 1 file changed, 2 deletions(-) delete mode 100755 salt/manager/tools/sbin/so-user-disable diff --git a/salt/manager/tools/sbin/so-user-disable b/salt/manager/tools/sbin/so-user-disable deleted file mode 100755 index c8049cdef..000000000 --- a/salt/manager/tools/sbin/so-user-disable +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -so-user disable --email $1 \ No newline at end of file From 69be03f86a25aaf0ffd6880a42518d39f1851309 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 2 Jul 2024 14:36:36 -0400 Subject: [PATCH 32/80] Delete salt/manager/tools/sbin/so-user-enable --- salt/manager/tools/sbin/so-user-enable | 2 -- 1 file changed, 2 deletions(-) delete mode 100755 salt/manager/tools/sbin/so-user-enable diff --git a/salt/manager/tools/sbin/so-user-enable b/salt/manager/tools/sbin/so-user-enable deleted file mode 100755 index 54ddca6f0..000000000 --- a/salt/manager/tools/sbin/so-user-enable +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -so-user enable --email $1 \ No newline at end of file From 40b7999786bbb6cce99fcf6361030f822cebb9d2 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Tue, 2 Jul 2024 14:36:51 -0400 Subject: [PATCH 33/80] Delete salt/manager/tools/sbin/so-user-list --- salt/manager/tools/sbin/so-user-list | 2 -- 1 file changed, 2 deletions(-) delete mode 100755 salt/manager/tools/sbin/so-user-list diff --git a/salt/manager/tools/sbin/so-user-list b/salt/manager/tools/sbin/so-user-list deleted file mode 100755 index a156ffe5f..000000000 --- a/salt/manager/tools/sbin/so-user-list +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -so-user list \ No newline at end of file From c6f6811f47d016f648c0e4e4ae1ea126c3c95eb0 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 Jul 2024 09:47:34 -0400 Subject: [PATCH 34/80] Elastic 8.14.2 --- salt/common/tools/sbin/so-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 02d16ad2c..8a6effa5c 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -8,7 +8,7 @@ # Elastic agent is not managed by salt. Because of this we must store this base information in a # script that accompanies the soup system. Since so-common is one of those special soup files, # and since this same logic is required during installation, it's included in this file. -ELASTIC_AGENT_TARBALL_VERSION="8.14.1" +ELASTIC_AGENT_TARBALL_VERSION="8.14.2" ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" From ef5967844137ca24ee2d804855ad7e61fd03a322 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 Jul 2024 09:48:12 -0400 Subject: [PATCH 35/80] Elastic 8.14.2 --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index f3370d053..a25bee36c 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.1","id": "8.14.1","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.2","id": "8.14.2,"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From 9504f0885ac275cd5bd324835141c7ccc56558d6 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 Jul 2024 09:49:07 -0400 Subject: [PATCH 36/80] Elastic 8.14.2 --- salt/kibana/tools/sbin_jinja/so-kibana-config-load | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index dd36fd030..fc0896009 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.1" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From 6a396ec1aa8c49cd483ef0462b5078c8c96e56d0 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 Jul 2024 11:44:27 -0400 Subject: [PATCH 37/80] Fix accidental double quote removal --- salt/kibana/files/config_saved_objects.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index a25bee36c..50e2ba45c 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.2","id": "8.14.2,"references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.2","id": "8.14.2","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} From 2dd5ff433345b253c7b03e951e35e8867950a800 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 Jul 2024 16:19:53 -0400 Subject: [PATCH 38/80] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7d52aac7f..38f621b25 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-foxtrot +2.4.90 From 8615e5d5eab8305f337be71e6ef085ace8a75bf9 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 8 Jul 2024 16:50:06 -0400 Subject: [PATCH 39/80] Move enabled and index_clean back to the top --- salt/elasticsearch/defaults.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 98eac89f2..e1a2d192f 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,4 +1,6 @@ elasticsearch: + enabled: false + index_clean: true config: action: destructive_requires_name: true @@ -54,8 +56,6 @@ elasticsearch: enabled: true key: /usr/share/elasticsearch/config/elasticsearch.key verification_mode: none - enabled: false - index_clean: true index_settings: global_overrides: index_template: From 24e945eee493790153f3b35a2903fe0d2f57c161 Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 9 Jul 2024 10:06:16 -0400 Subject: [PATCH 40/80] FIX: Update MOTD #13317 --- salt/motd/files/so_motd.jinja | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/salt/motd/files/so_motd.jinja b/salt/motd/files/so_motd.jinja index 2997629bc..721b10d40 100644 --- a/salt/motd/files/so_motd.jinja +++ b/salt/motd/files/so_motd.jinja @@ -15,12 +15,11 @@ Access the Security Onion web interface at https://{{ GLOBALS.url_base }} {%- endfor -%} {%- if minions_need_restarted | length > 0 %} -**************************************************************************************************** -* The following nodes in your Security Onion grid may need to be restarted due to package updates. * -* If the node has already been patched, restarted and been up for less than 15 minutes, then it * -* may not have updated it's restart_needed status yet. This will cause it to be listed below, even * -* if it has already been restarted. This feature will be improved in the future. * -**************************************************************************************************** +#################################################################################################### +# The following nodes in your Security Onion grid may need to be restarted due to package updates. # +# If a node has already been patched and restarted but has been up for less than 15 minutes, # +# then it may not have updated its status yet. # +#################################################################################################### {% for minion in minions_need_restarted -%} {{ minion }} From 669f68ad88c801f64d3d8d544aed6a749f1113e6 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 9 Jul 2024 15:39:59 +0000 Subject: [PATCH 41/80] Fleet metric annotations --- salt/elasticsearch/soc_elasticsearch.yaml | 70 +++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index f56ed313e..47beb27f8 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -530,6 +530,76 @@ elasticsearch: so-strelka: *indexSettings so-syslog: *indexSettings so-zeek: *indexSettings + so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings + index_sorting: + description: Sorts the index by event time, at the cost of additional processing resource consumption. + advanced: True + readonly: True + helpLink: elasticsearch.html + index_template: + ignore_missing_component_templates: + description: Ignore component templates if they aren't in Elasticsearch. + forcedType: "[]string" + multiline: True + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + index_patterns: + description: Patterns for matching multiple indices or tables. + forceType: "[]string" + multiline: True + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + template: + settings: + index: + mode: + description: Type of mode used for this index. Time series indices can be used for metrics to reduce necessary storage. + forcedType: string + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + number_of_replicas: + description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. + forcedType: int + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + composed_of: + description: The index template is composed of these component templates. + forcedType: "[]string" + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + priority: + description: The priority of the index template. + forcedType: int + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + data_stream: + hidden: + description: Hide the data stream. + forcedType: bool + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + allow_custom_routing: + description: Allow custom routing for the data stream. + forcedType: bool + global: True + advanced: True + readonly: True + helpLink: elasticsearch.html + so-metrics-fleet_server_x_agent_versions: *fleetMetricsSettings so_roles: so-manager: &soroleSettings config: From 9716e09b83a6f9de565708da3a8ca1fb3ae7de81 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 9 Jul 2024 12:51:34 -0400 Subject: [PATCH 42/80] Temp change for testing --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 38f621b25..7d52aac7f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.90 +2.4.0-foxtrot From dfd8ac3626de18179bb01f93d31cd888171282ae Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Tue, 9 Jul 2024 12:55:58 -0400 Subject: [PATCH 43/80] FIX: Update SOC MOTD #13320 --- salt/soc/files/soc/motd.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/salt/soc/files/soc/motd.md b/salt/soc/files/soc/motd.md index c13cbd70b..91c603851 100644 --- a/salt/soc/files/soc/motd.md +++ b/salt/soc/files/soc/motd.md @@ -1,6 +1,6 @@ ## Getting Started -New to Security Onion? Click the menu in the upper-right corner and you'll find links for [Help](/docs/) and a [Cheat Sheet](/docs/cheatsheet.pdf) that will help you best utilize Security Onion to hunt for evil! In addition, check out our free Security Onion Essentials online course, available on our [Training](https://securityonionsolutions.com/training) website. +New to Security Onion? Click the menu in the upper-right corner and you'll find links for [Help](/docs/) and a [Cheat Sheet](/docs/cheatsheet.pdf) that will help you best utilize Security Onion to hunt for evil! In addition, check out our free Security Onion Essentials online course, available on our [Training](https://securityonion.com/training) website. If you're ready to dive in, take a look at the [Alerts](/#/alerts) interface to see what Security Onion has detected so far. If you find any false positives, then you can tune those in [Detections](/#/detections). @@ -20,13 +20,17 @@ For more coverage of your enterprise, you can deploy the Elastic Agent to endpoi To see all the latest features and fixes in this version of Security Onion, click the upper-right menu and then click the [What's New](/docs/release-notes.html) link. +## Security Onion Pro + +Need enterprise features and premium support? Check out [Security Onion Pro](https://securityonion.com/pro/)! + ## Enterprise Appliances -Want the best hardware for your enterprise deployment? Check out our [enterprise appliances](https://securityonionsolutions.com/hardware/)! +Want the best hardware for your enterprise deployment? Check out our [enterprise appliances](https://securityonion.com/hardware/)! ## Premium Support -Experiencing difficulties and need priority support or remote assistance? We offer a [premium support plan](https://securityonionsolutions.com/support/) to assist corporate, educational, and government organizations. +Experiencing difficulties and need priority support or remote assistance? We offer a [premium support plan](https://securityonion.com/support/) to assist corporate, educational, and government organizations. ## Customize This Space From 7a2f01be5373f214e896cc8250ad632ee300dcd9 Mon Sep 17 00:00:00 2001 From: weslambert Date: Tue, 9 Jul 2024 13:58:13 -0400 Subject: [PATCH 44/80] Update VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7d52aac7f..38f621b25 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.0-foxtrot +2.4.90 From 1f5a990b1ee9ae184fc891fea29890d834d66962 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 9 Jul 2024 18:32:06 +0000 Subject: [PATCH 45/80] Remove lines that aren't needed right now --- salt/elasticsearch/soc_elasticsearch.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 47beb27f8..085aab7f0 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -539,17 +539,11 @@ elasticsearch: index_template: ignore_missing_component_templates: description: Ignore component templates if they aren't in Elasticsearch. - forcedType: "[]string" - multiline: True - global: True advanced: True readonly: True helpLink: elasticsearch.html index_patterns: description: Patterns for matching multiple indices or tables. - forceType: "[]string" - multiline: True - global: True advanced: True readonly: True helpLink: elasticsearch.html @@ -558,44 +552,32 @@ elasticsearch: index: mode: description: Type of mode used for this index. Time series indices can be used for metrics to reduce necessary storage. - forcedType: string - global: True advanced: True readonly: True helpLink: elasticsearch.html number_of_replicas: description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. - forcedType: int - global: True advanced: True readonly: True helpLink: elasticsearch.html composed_of: description: The index template is composed of these component templates. - forcedType: "[]string" - global: True advanced: True readonly: True helpLink: elasticsearch.html priority: description: The priority of the index template. - forcedType: int - global: True advanced: True readonly: True helpLink: elasticsearch.html data_stream: hidden: description: Hide the data stream. - forcedType: bool - global: True advanced: True readonly: True helpLink: elasticsearch.html allow_custom_routing: description: Allow custom routing for the data stream. - forcedType: bool - global: True advanced: True readonly: True helpLink: elasticsearch.html From d791b23838c5edd4f407256eccd04db70dc60428 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:29:09 -0400 Subject: [PATCH 46/80] Generate new Kafka truststore Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- pillar/top.sls | 2 + salt/allowed_states.map.jinja | 4 +- salt/kafka/ca.sls | 37 +++++++++++++++++++ salt/kafka/config.map.jinja | 16 +++----- salt/kafka/config.sls | 22 +++++++++-- salt/kafka/defaults.yaml | 21 ++++++----- salt/kafka/enabled.sls | 10 +++-- salt/kafka/soc_kafka.yaml | 24 +++++++++--- salt/kafka/tools/sbin_jinja/so-kafka-trust | 13 +++++++ .../config/so/0800_input_kafka.conf.jinja | 5 ++- salt/manager/tools/sbin/soup | 5 ++- setup/so-functions | 4 +- 12 files changed, 125 insertions(+), 38 deletions(-) create mode 100644 salt/kafka/ca.sls create mode 100644 salt/kafka/tools/sbin_jinja/so-kafka-trust diff --git a/pillar/top.sls b/pillar/top.sls index 14229162f..76d1a14e1 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -233,6 +233,8 @@ base: - stig.soc_stig - soc.license - kafka.nodes + - kafka.soc_kafka + - kafka.adv_kafka '*_receiver': - logstash.nodes diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index eb73e6e29..29ee968aa 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -136,7 +136,9 @@ 'firewall', 'schedule', 'docker_clean', - 'stig' + 'stig', + 'kafka.ca', + 'kafka.ssl' ], 'so-standalone': [ 'salt.master', diff --git a/salt/kafka/ca.sls b/salt/kafka/ca.sls new file mode 100644 index 000000000..f5e78ee2c --- /dev/null +++ b/salt/kafka/ca.sls @@ -0,0 +1,37 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls.split('.')[0] in allowed_states or sls in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% set KAFKATRUST = salt['pillar.get']('kafka:truststore') %} + +kafkaconfdir: + file.directory: + - name: /opt/so/conf/kafka + - user: 960 + - group: 960 + - makedirs: True + +{% if GLOBALS.is_manager %} +# Manager runs so-kafka-trust to create truststore for Kafka ssl communication +kafka_truststore: + cmd.script: + - source: salt://kafka/tools/sbin_jinja/so-kafka-trust + - template: jinja + - cwd: /opt/so + - defaults: + GLOBALS: {{ GLOBALS }} + KAFKATRUST: {{ KAFKATRUST }} +{% endif %} + +kafkacertz: + file.managed: + - name: /opt/so/conf/kafka/kafka-truststore.jks + - source: salt://kafka/files/kafka-truststore + - user: 960 + - group: 931 + +{% endif %} \ No newline at end of file diff --git a/salt/kafka/config.map.jinja b/salt/kafka/config.map.jinja index 4c408a1e7..b054e0656 100644 --- a/salt/kafka/config.map.jinja +++ b/salt/kafka/config.map.jinja @@ -7,6 +7,7 @@ {% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %} {% set KAFKA_PASSWORD = salt['pillar.get']('kafka:password') %} +{% set KAFKA_TRUSTPASS = salt['pillar.get']('kafka:trustpass') %} {# Create list of KRaft controllers #} {% set controllers = [] %} @@ -67,19 +68,12 @@ {% endif %} -{# If a password other than PLACEHOLDER isn't set remove it from the server.properties #} -{% if KAFKAMERGED.config.broker.ssl_x_truststore_x_password == 'PLACEHOLDER' %} -{% do KAFKAMERGED.config.broker.pop('ssl_x_truststore_x_password') %} -{% endif %} - -{% if KAFKAMERGED.config.controller.ssl_x_truststore_x_password == 'PLACEHOLDER' %} -{% do KAFKAMERGED.config.controller.pop('ssl_x_truststore_x_password') %} -{% endif %} +{# Truststore config #} +{% do KAFKAMERGED.config.broker.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %} +{% do KAFKAMERGED.config.controller.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %} +{% do KAFKAMERGED.config.client.update({'ssl_x_truststore_x_password': KAFKA_TRUSTPASS }) %} {# Client properties stuff #} -{% if KAFKAMERGED.config.client.ssl_x_truststore_x_password == 'PLACEHOLDER' %} -{% do KAFKAMERGED.config.client.pop('ssl_x_truststore_x_password') %} -{% endif %} {% do KAFKAMERGED.config.client.update({'ssl_x_keystore_x_password': KAFKA_PASSWORD }) %} {% if 'broker' in node_type %} diff --git a/salt/kafka/config.sls b/salt/kafka/config.sls index 6293ee697..1cfd1d3eb 100644 --- a/salt/kafka/config.sls +++ b/salt/kafka/config.sls @@ -7,18 +7,21 @@ {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} -include: - - ssl - kafka_group: group.present: - name: kafka - gid: 960 -kafka: +kafka_user: user.present: - uid: 960 - gid: 960 + - home: /opt/so/conf/kafka + - createhome: False + +kafka_home_dir: + file.absent: + - name: /home/kafka kafka_sbin_tools: file.recurse: @@ -28,6 +31,17 @@ kafka_sbin_tools: - group: 960 - file_mode: 755 +kafka_sbin_jinja_tools: + file.recurse: + - name: /usr/sbin + - source: salt://kafka/tools/sbin_jinja + - user: 960 + - group: 960 + - file_mode: 755 + - template: jinja + - defaults: + GLOBALS: {{ GLOBALS }} + kafka_log_dir: file.directory: - name: /opt/so/log/kafka diff --git a/salt/kafka/defaults.yaml b/salt/kafka/defaults.yaml index ad626458f..c20d8552c 100644 --- a/salt/kafka/defaults.yaml +++ b/salt/kafka/defaults.yaml @@ -1,10 +1,11 @@ kafka: enabled: False cluster_id: - password: controllers: reset: config: + password: + trustpass: broker: advertised_x_listeners: auto_x_create_x_topics_x_enable: true @@ -30,16 +31,16 @@ kafka: ssl_x_keystore_x_location: /etc/pki/kafka.p12 ssl_x_keystore_x_type: PKCS12 ssl_x_keystore_x_password: - ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts - ssl_x_truststore_x_password: PLACEHOLDER - ssl_x_truststore_x_type: PEM + ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks + ssl_x_truststore_x_type: JKS + ssl_x_truststore_x_password: transaction_x_state_x_log_x_min_x_isr: 1 transaction_x_state_x_log_x_replication_x_factor: 1 client: security_x_protocol: SSL - ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts - ssl_x_truststore_x_password: PLACEHOLDER - ssl_x_truststore_x_type: PEM + ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks + ssl_x_truststore_x_type: JKS + ssl_x_truststore_x_password: ssl_x_keystore_x_location: /etc/pki/kafka.p12 ssl_x_keystore_x_type: PKCS12 ssl_x_keystore_x_password: @@ -57,6 +58,6 @@ kafka: ssl_x_keystore_x_location: /etc/pki/kafka.p12 ssl_x_keystore_x_type: PKCS12 ssl_x_keystore_x_password: - ssl_x_truststore_x_location: /etc/pki/java/sos/cacerts - ssl_x_truststore_x_password: PLACEHOLDER - ssl_x_truststore_x_type: PEM \ No newline at end of file + ssl_x_truststore_x_location: /etc/pki/kafka-truststore.jks + ssl_x_truststore_x_type: JKS + ssl_x_truststore_x_password: \ No newline at end of file diff --git a/salt/kafka/enabled.sls b/salt/kafka/enabled.sls index 0837b5af6..362f7fde3 100644 --- a/salt/kafka/enabled.sls +++ b/salt/kafka/enabled.sls @@ -17,10 +17,11 @@ {% if 'gmd' in salt['pillar.get']('features', []) %} include: - - elasticsearch.ca - - kafka.sostatus + - kafka.ca - kafka.config + - kafka.ssl - kafka.storage + - kafka.sostatus so-kafka: docker_container.running: @@ -49,7 +50,7 @@ so-kafka: {% endfor %} - binds: - /etc/pki/kafka.p12:/etc/pki/kafka.p12:ro - - /etc/pki/tls/certs/intca.crt:/etc/pki/java/sos/cacerts:ro + - /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro - /nsm/kafka/data/:/nsm/kafka/data/:rw - /opt/so/log/kafka:/opt/kafka/logs/:rw - /opt/so/conf/kafka/server.properties:/opt/kafka/config/kraft/server.properties:ro @@ -58,6 +59,9 @@ so-kafka: {% for sc in ['server', 'client'] %} - file: kafka_kraft_{{sc}}_properties {% endfor %} + - file: kafkacertz + - require: + - file: kafkacertz delete_so-kafka_so-status.disabled: file.uncomment: diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml index 0c9c8a57e..872bf51f2 100644 --- a/salt/kafka/soc_kafka.yaml +++ b/salt/kafka/soc_kafka.yaml @@ -8,19 +8,25 @@ kafka: advanced: True sensitive: True helpLink: kafka.html - password: - description: The password to use for the Kafka certificates. - sensitive: True - helpLink: kafka.html controllers: description: A comma-separated list of hostnames that will act as Kafka controllers. These hosts will be responsible for managing the Kafka cluster. Note that only manager and receiver nodes are eligible to run Kafka. This configuration needs to be set before enabling Kafka. Failure to do so may result in Kafka topics becoming unavailable requiring manual intervention to restore functionality or reset Kafka, either of which can result in data loss. - forcedType: "string" + forcedType: string helpLink: kafka.html reset: description: Disable and reset the Kafka cluster. This will remove all Kafka data including logs that may have not yet been ingested into Elasticsearch and reverts the grid to using REDIS as the global pipeline. This is useful when testing different Kafka configurations such as rearranging Kafka brokers / controllers allowing you to reset the cluster rather than manually fixing any issues arising from attempting to reassign a Kafka broker into a controller. Enter 'YES_RESET_KAFKA' and submit to disable and reset Kafka. Make any configuration changes required and re-enable Kafka when ready. This action CANNOT be reversed. advanced: True helpLink: kafka.html config: + password: + description: The password used for the Kafka certificates. + readonly: True + sensitive: True + helpLink: kafka.html + trustpass: + description: The password used for the Kafka truststore. + readonly: True + sensitive: True + helpLink: kafka.html broker: advertised_x_listeners: description: Specify the list of listeners (hostname and port) that Kafka brokers provide to clients for communication. @@ -128,6 +134,10 @@ kafka: description: The trust store file location within the Docker container. title: ssl.truststore.location helpLink: kafka.html + ssl_x_truststore_x_type: + description: The trust store file format. + title: ssl.truststore.type + helpLink: kafka.html ssl_x_truststore_x_password: description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format. title: ssl.truststore.password @@ -167,6 +177,10 @@ kafka: description: The trust store file location within the Docker container. title: ssl.truststore.location helpLink: kafka.html + ssl_x_truststore_x_type: + description: The trust store file format. + title: ssl.truststore.type + helpLink: kafka.html ssl_x_truststore_x_password: description: The trust store file password. If null, the trust store file is still use, but integrity checking is disabled. Invalid for PEM format. title: ssl.truststore.password diff --git a/salt/kafka/tools/sbin_jinja/so-kafka-trust b/salt/kafka/tools/sbin_jinja/so-kafka-trust new file mode 100644 index 000000000..8d404cb9a --- /dev/null +++ b/salt/kafka/tools/sbin_jinja/so-kafka-trust @@ -0,0 +1,13 @@ +#!/bin/bash +# +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. +{% set TRUSTPASS = salt['pillar.get']('kafka:trustpass') %} + +if [ ! -f /opt/so/saltstack/local/salt/kafka/files/kafka-truststore ]; then + docker run -v /etc/pki/ca.crt:/etc/pki/ca.crt --name so-kafkatrust --user root --entrypoint /opt/java/openjdk/bin/keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} -import -file /etc/pki/ca.crt -alias SOS -keystore /etc/pki/kafka-truststore -storepass {{ TRUSTPASS }} -storetype jks -noprompt + docker cp so-kafkatrust:/etc/pki/kafka-truststore /opt/so/saltstack/local/salt/kafka/files/kafka-truststore + docker rm so-kafkatrust +fi \ No newline at end of file diff --git a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja index dfb246210..00dd6d530 100644 --- a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja +++ b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja @@ -1,4 +1,5 @@ {%- set kafka_password = salt['pillar.get']('kafka:password') %} +{%- set kafka_trustpass = salt['pillar.get']('kafka:trustpass') %} {%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %} {%- set brokers = [] %} @@ -22,8 +23,8 @@ input { ssl_keystore_location => '/usr/share/logstash/kafka-logstash.p12' ssl_keystore_password => '{{ kafka_password }}' ssl_keystore_type => 'PKCS12' - ssl_truststore_location => '/etc/pki/ca-trust/extracted/java/cacerts' - ssl_truststore_password => 'changeit' + ssl_truststore_location => '/etc/pki/kafka-truststore.jks' + ssl_truststore_password => '{{ kafka_trustpass }}' decorate_events => true tags => [ "elastic-agent", "input-{{ GLOBALS.hostname}}", "kafka" ] } diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index c76fe311e..019f29ebb 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -673,7 +673,10 @@ up_to_2.4.80() { } up_to_2.4.90() { - echo "Nothing to apply" + kafkatrust=$(get_random_value) + echo ' trustpass: '$kafkatrust >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls + + INSTALLEDVERSION=2.4.90 } diff --git a/setup/so-functions b/setup/so-functions index b1469b7eb..aa9eb1909 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1180,13 +1180,15 @@ kibana_pillar() { kafka_pillar() { KAFKACLUSTERID=$(get_random_value 22) KAFKAPASS=$(get_random_value) + KAFKATRUST=$(get_random_value) logCmd "mkdir -p $local_salt_dir/pillar/kafka" logCmd "touch $adv_kafka_pillar_file" logCmd "touch $kafka_pillar_file" printf '%s\n'\ "kafka:"\ " cluster_id: $KAFKACLUSTERID"\ - " password: $KAFKAPASS" > $kafka_pillar_file + " password: $KAFKAPASS"\ + " trustpass: $KAFKATRUST" > $kafka_pillar_file } logrotate_pillar() { From 8e1edd1d91e21aea68e6092f743917295d228786 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:32:43 -0400 Subject: [PATCH 47/80] split Kafka ssl from ssl/init. Certs won't be generated until Kafka is enabled. Also runs some clean up for old Kafka certs Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/kafka/disabled.sls | 11 ++- salt/kafka/ssl.sls | 196 ++++++++++++++++++++++++++++++++++++++ salt/logstash/enabled.sls | 14 ++- salt/ssl/init.sls | 190 +----------------------------------- 4 files changed, 221 insertions(+), 190 deletions(-) create mode 100644 salt/kafka/ssl.sls diff --git a/salt/kafka/disabled.sls b/salt/kafka/disabled.sls index 707e953a4..79fd0c261 100644 --- a/salt/kafka/disabled.sls +++ b/salt/kafka/disabled.sls @@ -22,4 +22,13 @@ ensure_default_pipeline: - name: | /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False; /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/global/soc_global.sls global.pipeline REDIS -{% endif %} \ No newline at end of file +{% endif %} + +{# If Kafka has never been manually enabled, the 'Kafka' user does not exist. In this case certs for Kafka should not exist since they'll be owned by uid 960 #} +{% for cert in ['kafka-client.crt','kafka-client.key','kafka.crt','kafka.key','kafka-logstash.crt','kafka-logstash.key','kafka-logstash.p12','kafka.p12','elasticfleet-kafka.p8'] %} +check_kafka_cert_{{cert}}: + file.absent: + - name: /etc/pki/{{cert}} + - onlyif: stat -c %U /etc/pki/{{cert}} | grep -q UNKNOWN + - show_changes: False +{% endfor %} \ No newline at end of file diff --git a/salt/kafka/ssl.sls b/salt/kafka/ssl.sls new file mode 100644 index 000000000..c4e46ac8a --- /dev/null +++ b/salt/kafka/ssl.sls @@ -0,0 +1,196 @@ +# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# https://securityonion.net/license; you may not use this file except in compliance with the +# Elastic License 2.0. + +{% from 'allowed_states.map.jinja' import allowed_states %} +{% if sls in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} + +{% set kafka_password = salt['pillar.get']('kafka:password') %} + +include: + - ca.dirs + {% set global_ca_server = [] %} + {% set x509dict = salt['mine.get'](GLOBALS.manager | lower~'*', 'x509.get_pem_entries') %} + {% for host in x509dict %} + {% if 'manager' in host.split('_')|last or host.split('_')|last == 'standalone' %} + {% do global_ca_server.append(host) %} + {% endif %} + {% endfor %} + {% set ca_server = global_ca_server[0] %} + + +{% if GLOBALS.pipeline == "KAFKA" %} + +{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] %} +kafka_client_key: + x509.private_key_managed: + - name: /etc/pki/kafka-client.key + - keysize: 4096 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/kafka-client.key') -%} + - prereq: + - x509: /etc/pki/kafka-client.crt + {%- endif %} + - retry: + attempts: 5 + interval: 30 + +kafka_client_crt: + x509.certificate_managed: + - name: /etc/pki/kafka-client.crt + - ca_server: {{ ca_server }} + - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} + - signing_policy: kafka + - private_key: /etc/pki/kafka-client.key + - CN: {{ GLOBALS.hostname }} + - days_remaining: 0 + - days_valid: 820 + - backup: True + - timeout: 30 + - retry: + attempts: 5 + interval: 30 + +kafka_client_key_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka-client.key + - mode: 640 + - user: 960 + - group: 939 + +kafka_client_crt_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka-client.crt + - mode: 640 + - user: 960 + - group: 939 +{% endif %} + +{% if grains['role'] in ['so-manager', 'so-managersearch','so-receiver', 'so-standalone'] %} +kafka_key: + x509.private_key_managed: + - name: /etc/pki/kafka.key + - keysize: 4096 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/kafka.key') -%} + - prereq: + - x509: /etc/pki/kafka.crt + {%- endif %} + - retry: + attempts: 5 + interval: 30 + +kafka_crt: + x509.certificate_managed: + - name: /etc/pki/kafka.crt + - ca_server: {{ ca_server }} + - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} + - signing_policy: kafka + - private_key: /etc/pki/kafka.key + - CN: {{ GLOBALS.hostname }} + - days_remaining: 0 + - days_valid: 820 + - backup: True + - timeout: 30 + - retry: + attempts: 5 + interval: 30 + cmd.run: + - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka.key -in /etc/pki/kafka.crt -export -out /etc/pki/kafka.p12 -nodes -passout pass:{{ kafka_password }}" + - onchanges: + - x509: /etc/pki/kafka.key +kafka_key_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka.key + - mode: 640 + - user: 960 + - group: 939 + +kafka_crt_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka.crt + - mode: 640 + - user: 960 + - group: 939 + +kafka_pkcs12_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka.p12 + - mode: 640 + - user: 960 + - group: 939 +{% endif %} + +# Standalone needs kafka-logstash for automated testing. Searchnode/manager search need it for logstash to consume from Kafka. +# Manager will have cert, but be unused until a pipeline is created and logstash enabled. +{% if grains['role'] in ['so-standalone', 'so-managersearch', 'so-searchnode', 'so-manager'] %} +kafka_logstash_key: + x509.private_key_managed: + - name: /etc/pki/kafka-logstash.key + - keysize: 4096 + - backup: True + - new: True + {% if salt['file.file_exists']('/etc/pki/kafka-logstash.key') -%} + - prereq: + - x509: /etc/pki/kafka-logstash.crt + {%- endif %} + - retry: + attempts: 5 + interval: 30 + +kafka_logstash_crt: + x509.certificate_managed: + - name: /etc/pki/kafka-logstash.crt + - ca_server: {{ ca_server }} + - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} + - signing_policy: kafka + - private_key: /etc/pki/kafka-logstash.key + - CN: {{ GLOBALS.hostname }} + - days_remaining: 0 + - days_valid: 820 + - backup: True + - timeout: 30 + - retry: + attempts: 5 + interval: 30 + cmd.run: + - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka-logstash.key -in /etc/pki/kafka-logstash.crt -export -out /etc/pki/kafka-logstash.p12 -nodes -passout pass:{{ kafka_password }}" + - onchanges: + - x509: /etc/pki/kafka-logstash.key + +kafka_logstash_key_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka-logstash.key + - mode: 640 + - user: 931 + - group: 939 + +kafka_logstash_crt_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka-logstash.crt + - mode: 640 + - user: 931 + - group: 939 + +kafka_logstash_pkcs12_perms: + file.managed: + - replace: False + - name: /etc/pki/kafka-logstash.p12 + - mode: 640 + - user: 931 + - group: 939 + +{% endif %} + +{% endif %} \ No newline at end of file diff --git a/salt/logstash/enabled.sls b/salt/logstash/enabled.sls index f95a76f13..0f44a3767 100644 --- a/salt/logstash/enabled.sls +++ b/salt/logstash/enabled.sls @@ -14,6 +14,11 @@ include: {% if GLOBALS.role not in ['so-receiver','so-fleet'] %} - elasticsearch.ca +{% endif %} +{# Kafka ca runs on nodes that can run logstash for Kafka input / output. Only when Kafka is global pipeline #} +{% if GLOBALS.role in ['so-searchnode', 'so-manager', 'so-managersearch', 'so-receiver', 'so-standalone'] and GLOBALS.pipeline == 'KAFKA' %} + - kafka.ca + - kafka.ssl {% endif %} - logstash.config - logstash.sostatus @@ -79,8 +84,9 @@ so-logstash: - /opt/so/conf/ca/cacerts:/etc/pki/ca-trust/extracted/java/cacerts:ro - /opt/so/conf/ca/tls-ca-bundle.pem:/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem:ro {% endif %} - {% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %} + {% if GLOBALS.pipeline == "KAFKA" and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %} - /etc/pki/kafka-logstash.p12:/usr/share/logstash/kafka-logstash.p12:ro + - /opt/so/conf/kafka/kafka-truststore.jks:/etc/pki/kafka-truststore.jks:ro {% endif %} {% if GLOBALS.role == 'so-eval' %} - /nsm/zeek:/nsm/zeek:ro @@ -105,6 +111,9 @@ so-logstash: - file: ls_pipeline_{{assigned_pipeline}}_{{CONFIGFILE.split('.')[0] | replace("/","_") }} {% endfor %} {% endfor %} + {% if GLOBALS.pipeline == 'KAFKA' and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %} + - file: kafkacertz + {% endif %} - require: {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone', 'so-import', 'so-heavynode', 'so-receiver'] %} - x509: etc_filebeat_crt @@ -118,6 +127,9 @@ so-logstash: - file: cacertz - file: capemz {% endif %} + {% if GLOBALS.pipeline == 'KAFKA' and GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-searchnode'] %} + - file: kafkacertz + {% endif %} delete_so-logstash_so-status.disabled: file.uncomment: diff --git a/salt/ssl/init.sls b/salt/ssl/init.sls index abcb1a559..f5be34c40 100644 --- a/salt/ssl/init.sls +++ b/salt/ssl/init.sls @@ -17,8 +17,6 @@ {% set COMMONNAME = GLOBALS.manager %} {% endif %} -{% set kafka_password = salt['pillar.get']('kafka:password') %} - {% if grains.id.split('_')|last in ['manager', 'managersearch', 'eval', 'standalone', 'import'] %} include: - ca @@ -666,7 +664,6 @@ elastickeyperms: {%- endif %} {% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] %} - elasticfleet_kafka_key: x509.private_key_managed: - name: /etc/pki/elasticfleet-kafka.key @@ -696,17 +693,13 @@ elasticfleet_kafka_crt: - retry: attempts: 5 interval: 30 - cmd.run: - - name: "/usr/bin/openssl pkcs8 -in /etc/pki/elasticfleet-kafka.key -topk8 -out /etc/pki/elasticfleet-kafka.p8 -nocrypt" - - onchanges: - - x509: elasticfleet_kafka_key elasticfleet_kafka_cert_perms: file.managed: - replace: False - name: /etc/pki/elasticfleet-kafka.crt - mode: 640 - - user: 960 + - user: 947 - group: 939 elasticfleet_kafka_key_perms: @@ -714,187 +707,8 @@ elasticfleet_kafka_key_perms: - replace: False - name: /etc/pki/elasticfleet-kafka.key - mode: 640 - - user: 960 + - user: 947 - group: 939 - -elasticfleet_kafka_pkcs8_perms: - file.managed: - - replace: False - - name: /etc/pki/elasticfleet-kafka.p8 - - mode: 640 - - user: 960 - - group: 939 - -kafka_client_key: - x509.private_key_managed: - - name: /etc/pki/kafka-client.key - - keysize: 4096 - - backup: True - - new: True - {% if salt['file.file_exists']('/etc/pki/kafka-client.key') -%} - - prereq: - - x509: /etc/pki/kafka-client.crt - {%- endif %} - - retry: - attempts: 5 - interval: 30 - -kafka_client_crt: - x509.certificate_managed: - - name: /etc/pki/kafka-client.crt - - ca_server: {{ ca_server }} - - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - - signing_policy: kafka - - private_key: /etc/pki/kafka-client.key - - CN: {{ GLOBALS.hostname }} - - days_remaining: 0 - - days_valid: 820 - - backup: True - - timeout: 30 - - retry: - attempts: 5 - interval: 30 - -kafka_client_key_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka-client.key - - mode: 640 - - user: 960 - - group: 939 - -kafka_client_crt_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka-client.crt - - mode: 640 - - user: 960 - - group: 939 - -{% endif %} - -{% if grains['role'] in ['so-manager', 'so-managersearch','so-receiver', 'so-standalone'] %} - -kafka_key: - x509.private_key_managed: - - name: /etc/pki/kafka.key - - keysize: 4096 - - backup: True - - new: True - {% if salt['file.file_exists']('/etc/pki/kafka.key') -%} - - prereq: - - x509: /etc/pki/kafka.crt - {%- endif %} - - retry: - attempts: 5 - interval: 30 - -kafka_crt: - x509.certificate_managed: - - name: /etc/pki/kafka.crt - - ca_server: {{ ca_server }} - - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - - signing_policy: kafka - - private_key: /etc/pki/kafka.key - - CN: {{ GLOBALS.hostname }} - - days_remaining: 0 - - days_valid: 820 - - backup: True - - timeout: 30 - - retry: - attempts: 5 - interval: 30 - cmd.run: - - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka.key -in /etc/pki/kafka.crt -export -out /etc/pki/kafka.p12 -nodes -passout pass:{{ kafka_password }}" - - onchanges: - - x509: /etc/pki/kafka.key -kafka_key_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka.key - - mode: 640 - - user: 960 - - group: 939 - -kafka_crt_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka.crt - - mode: 640 - - user: 960 - - group: 939 - -kafka_pkcs12_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka.p12 - - mode: 640 - - user: 960 - - group: 939 - -{% endif %} - -# Standalone needs kafka-logstash for automated testing. Searchnode/manager search need it for logstash to consume from Kafka. -# Manager will have cert, but be unused until a pipeline is created and logstash enabled. -{% if grains['role'] in ['so-standalone', 'so-managersearch', 'so-searchnode', 'so-manager'] %} -kafka_logstash_key: - x509.private_key_managed: - - name: /etc/pki/kafka-logstash.key - - keysize: 4096 - - backup: True - - new: True - {% if salt['file.file_exists']('/etc/pki/kafka-logstash.key') -%} - - prereq: - - x509: /etc/pki/kafka-logstash.crt - {%- endif %} - - retry: - attempts: 5 - interval: 30 - -kafka_logstash_crt: - x509.certificate_managed: - - name: /etc/pki/kafka-logstash.crt - - ca_server: {{ ca_server }} - - subjectAltName: DNS:{{ GLOBALS.hostname }}, IP:{{ GLOBALS.node_ip }} - - signing_policy: kafka - - private_key: /etc/pki/kafka-logstash.key - - CN: {{ GLOBALS.hostname }} - - days_remaining: 0 - - days_valid: 820 - - backup: True - - timeout: 30 - - retry: - attempts: 5 - interval: 30 - cmd.run: - - name: "/usr/bin/openssl pkcs12 -inkey /etc/pki/kafka-logstash.key -in /etc/pki/kafka-logstash.crt -export -out /etc/pki/kafka-logstash.p12 -nodes -passout pass:{{ kafka_password }}" - - onchanges: - - x509: /etc/pki/kafka-logstash.key - -kafka_logstash_key_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka-logstash.key - - mode: 640 - - user: 960 - - group: 939 - -kafka_logstash_crt_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka-logstash.crt - - mode: 640 - - user: 960 - - group: 939 - -kafka_logstash_pkcs12_perms: - file.managed: - - replace: False - - name: /etc/pki/kafka-logstash.p12 - - mode: 640 - - user: 960 - - group: 931 - {% endif %} {% else %} From d5faf535c3adf6101d2f7ad6c1498956aeba0a10 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:36:44 -0400 Subject: [PATCH 48/80] Only interact with logstash configuration when Kafka pipeline is enabled otherwise leave it default Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/kafka/defaults.yaml | 1 + salt/kafka/soc_kafka.yaml | 6 ++++++ salt/logstash/defaults.yaml | 3 +-- salt/logstash/init.sls | 6 +----- salt/logstash/map.jinja | 12 ++++++++++++ 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/salt/kafka/defaults.yaml b/salt/kafka/defaults.yaml index c20d8552c..21d6956ba 100644 --- a/salt/kafka/defaults.yaml +++ b/salt/kafka/defaults.yaml @@ -3,6 +3,7 @@ kafka: cluster_id: controllers: reset: + logstash: [] config: password: trustpass: diff --git a/salt/kafka/soc_kafka.yaml b/salt/kafka/soc_kafka.yaml index 872bf51f2..8087f9bdf 100644 --- a/salt/kafka/soc_kafka.yaml +++ b/salt/kafka/soc_kafka.yaml @@ -16,6 +16,12 @@ kafka: description: Disable and reset the Kafka cluster. This will remove all Kafka data including logs that may have not yet been ingested into Elasticsearch and reverts the grid to using REDIS as the global pipeline. This is useful when testing different Kafka configurations such as rearranging Kafka brokers / controllers allowing you to reset the cluster rather than manually fixing any issues arising from attempting to reassign a Kafka broker into a controller. Enter 'YES_RESET_KAFKA' and submit to disable and reset Kafka. Make any configuration changes required and re-enable Kafka when ready. This action CANNOT be reversed. advanced: True helpLink: kafka.html + logstash: + description: By default logstash is disabled when Kafka is enabled. This option allows you to specify any hosts you would like to re-enable logstash on alongside Kafka. + forcedType: "[]string" + multiline: True + advanced: True + helpLink: kafka.html config: password: description: The password used for the Kafka certificates. diff --git a/salt/logstash/defaults.yaml b/salt/logstash/defaults.yaml index b4bc885f6..9930b7bcf 100644 --- a/salt/logstash/defaults.yaml +++ b/salt/logstash/defaults.yaml @@ -25,7 +25,7 @@ logstash: - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf.jinja - so/0013_input_lumberjack_fleet.conf - - so/9999_output_redis.conf.jinja + - so/9999_output_redis.conf.jinja receiver: - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf.jinja @@ -35,7 +35,6 @@ logstash: - so/0900_input_redis.conf.jinja - so/9805_output_elastic_agent.conf.jinja - so/9900_output_endgame.conf.jinja - - so/0800_input_kafka.conf.jinja custom0: [] custom1: [] custom2: [] diff --git a/salt/logstash/init.sls b/salt/logstash/init.sls index 3bc539b35..62b2a2ebb 100644 --- a/salt/logstash/init.sls +++ b/salt/logstash/init.sls @@ -4,13 +4,9 @@ # Elastic License 2.0. {% from 'logstash/map.jinja' import LOGSTASH_MERGED %} -{% from 'kafka/map.jinja' import KAFKAMERGED %} include: -{# Disable logstash when Kafka is enabled except when the role is standalone #} -{% if LOGSTASH_MERGED.enabled and grains.role == 'so-standalone' %} - - logstash.enabled -{% elif LOGSTASH_MERGED.enabled and not KAFKAMERGED.enabled %} +{% if LOGSTASH_MERGED.enabled %} - logstash.enabled {% else %} - logstash.disabled diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index 0bb6de2b7..2c3e02864 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -6,6 +6,7 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'logstash/defaults.yaml' as LOGSTASH_DEFAULTS %} {% set LOGSTASH_MERGED = salt['pillar.get']('logstash', LOGSTASH_DEFAULTS.logstash, merge=True) %} +{% set KAFKA_LOGSTASH = salt['pillar.get']('kafka:logstash', []) %} {# used to store the redis nodes that logstash needs to know about to pull from the queue #} {% set LOGSTASH_REDIS_NODES = [] %} @@ -30,3 +31,14 @@ {% endfor %} {% endfor %} +{# Append Kafka input pipeline when Kafka is enabled #} +{% if GLOBALS.pipeline == 'KAFKA' %} +{% do LOGSTASH_MERGED.defined_pipelines.search.append('so/0800_input_kafka.conf.jinja') %} +{% do LOGSTASH_MERGED.defined_pipelines.manager.append('so/0800_input_kafka.conf.jinja') %} +{# Disable logstash on manager & receiver nodes unless it has an override configured #} +{% if not KAFKA_LOGSTASH %} +{% if GLOBALS.role in ['so-manager', 'so-receiver'] and GLOBALS.hostname not in KAFKA_LOGSTASH %} +{% do LOGSTASH_MERGED.update({'enabled': False}) %} +{% endif %} +{% endif %} +{% endif %} \ No newline at end of file From 0d8fd42be3e6c79f2a7f3e32fec01bc8291e81f2 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:37:07 -0400 Subject: [PATCH 49/80] update pillarwatch engine Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/salt/files/engines.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/salt/files/engines.conf b/salt/salt/files/engines.conf index 3a9b51207..15d55e18f 100644 --- a/salt/salt/files/engines.conf +++ b/salt/salt/files/engines.conf @@ -43,20 +43,20 @@ engines: - cmd.run: cmd: /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled True - cmd.run: - cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' saltutil.kill_all_jobs + cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' saltutil.kill_all_jobs - cmd.run: cmd: salt-call state.apply kafka.nodes - cmd.run: - cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.highstate + cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' state.highstate 'KAFKA': to: 'REDIS': - cmd.run: cmd: /usr/sbin/so-yaml.py replace /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.enabled False - cmd.run: - cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' saltutil.kill_all_jobs + cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' saltutil.kill_all_jobs - cmd.run: - cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver' state.highstate + cmd: salt -C 'G@role:so-standalone or G@role:so-manager or G@role:so-managersearch or G@role:so-receiver or G@role:so-searchnode' state.highstate - files: - /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls - /opt/so/saltstack/local/pillar/kafka/adv_kafka.sls From d432019ad91bd86f65c247ed1ee43ba89808166a Mon Sep 17 00:00:00 2001 From: weslambert Date: Wed, 10 Jul 2024 12:48:08 -0400 Subject: [PATCH 50/80] Change version from 1.13.1 to 1.20.0 --- .../component/elastic-agent/logs-elastic_agent@package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json index 7bcc34de9..efd85bb4b 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json @@ -6,7 +6,7 @@ "name": "logs" }, "codec": "best_compression", - "default_pipeline": "logs-elastic_agent-1.13.1", + "default_pipeline": "logs-elastic_agent-1.20.0", "mapping": { "total_fields": { "limit": "10000" From cfe5c1d76a160dec4e7b1888b04bd26bd06eb977 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jul 2024 13:24:02 -0400 Subject: [PATCH 51/80] remove elasticsearch.ca from receiver allowed_states. Replaced by generated kafka trust Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/allowed_states.map.jinja | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 29ee968aa..37795e9d7 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -197,7 +197,6 @@ 'schedule', 'docker_clean', 'kafka', - 'elasticsearch.ca', 'stig' ], 'so-desktop': [ From 4a88dedcb88833907aa53aec7f44cac27e66ed50 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Wed, 10 Jul 2024 16:18:46 -0400 Subject: [PATCH 52/80] Fixin kafka.ssl state and include name for kafka_user Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/kafka/config.sls | 1 + salt/kafka/ssl.sls | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/salt/kafka/config.sls b/salt/kafka/config.sls index 1cfd1d3eb..e9222388b 100644 --- a/salt/kafka/config.sls +++ b/salt/kafka/config.sls @@ -14,6 +14,7 @@ kafka_group: kafka_user: user.present: + - name: kafka - uid: 960 - gid: 960 - home: /opt/so/conf/kafka diff --git a/salt/kafka/ssl.sls b/salt/kafka/ssl.sls index c4e46ac8a..50a01b22c 100644 --- a/salt/kafka/ssl.sls +++ b/salt/kafka/ssl.sls @@ -4,10 +4,9 @@ # Elastic License 2.0. {% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls in allowed_states %} -{% from 'vars/globals.map.jinja' import GLOBALS %} - -{% set kafka_password = salt['pillar.get']('kafka:password') %} +{% if sls.split('.')[0] in allowed_states %} +{% from 'vars/globals.map.jinja' import GLOBALS %} +{% set kafka_password = salt['pillar.get']('kafka:password') %} include: - ca.dirs @@ -20,10 +19,9 @@ include: {% endfor %} {% set ca_server = global_ca_server[0] %} - {% if GLOBALS.pipeline == "KAFKA" %} -{% if grains['role'] in ['so-manager', 'so-managersearch', 'so-standalone'] %} +{% if GLOBALS.role in ['so-manager', 'so-managersearch', 'so-standalone'] %} kafka_client_key: x509.private_key_managed: - name: /etc/pki/kafka-client.key @@ -71,7 +69,7 @@ kafka_client_crt_perms: - group: 939 {% endif %} -{% if grains['role'] in ['so-manager', 'so-managersearch','so-receiver', 'so-standalone'] %} +{% if GLOBALS.role in ['so-manager', 'so-managersearch','so-receiver', 'so-standalone'] %} kafka_key: x509.private_key_managed: - name: /etc/pki/kafka.key @@ -132,7 +130,7 @@ kafka_pkcs12_perms: # Standalone needs kafka-logstash for automated testing. Searchnode/manager search need it for logstash to consume from Kafka. # Manager will have cert, but be unused until a pipeline is created and logstash enabled. -{% if grains['role'] in ['so-standalone', 'so-managersearch', 'so-searchnode', 'so-manager'] %} +{% if GLOBALS.role in ['so-standalone', 'so-managersearch', 'so-searchnode', 'so-manager'] %} kafka_logstash_key: x509.private_key_managed: - name: /etc/pki/kafka-logstash.key @@ -191,6 +189,13 @@ kafka_logstash_pkcs12_perms: - user: 931 - group: 939 +{% endif %} {% endif %} +{% else %} + +{{sls}}_state_not_allowed: + test.fail_without_changes: + - name: {{sls}}_state_not_allowed + {% endif %} \ No newline at end of file From ff29d9ca51501ce2df877ed0d2b30754f6ef22ff Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:23:51 -0400 Subject: [PATCH 53/80] Update log-check to ignore kafka data directories Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/common/tools/sbin/so-log-check | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/common/tools/sbin/so-log-check b/salt/common/tools/sbin/so-log-check index cf1691589..2a86b9f2c 100755 --- a/salt/common/tools/sbin/so-log-check +++ b/salt/common/tools/sbin/so-log-check @@ -241,6 +241,7 @@ exclude_log "mysqld.log" # MySQL is removed as of 2.4.70, logs may still be on exclude_log "soctopus.log" # Soctopus is removed as of 2.4.70, logs may still be on disk exclude_log "agentstatus.log" # ignore this log since it tracks agents in error state exclude_log "detections_runtime-status_yara.log" # temporarily ignore this log until Detections is more stable +exclude_log "/nsm/kafka/data/" # ignore Kafka data directory from log check. for log_file in $(cat /tmp/log_check_files); do status "Checking log file $log_file" From 4182ff66a0368a181c79e12c145baa7e5e1c68e3 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Thu, 11 Jul 2024 16:37:16 -0400 Subject: [PATCH 54/80] rearrange kafka pillar, declutters SOC ui Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/kafka/config.map.jinja | 4 ++-- salt/kafka/ssl.sls | 2 +- salt/kafka/tools/sbin_jinja/so-kafka-trust | 2 +- .../pipelines/config/so/0800_input_kafka.conf.jinja | 4 ++-- salt/manager/tools/sbin/soup | 8 +++++++- setup/so-functions | 5 +++-- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/salt/kafka/config.map.jinja b/salt/kafka/config.map.jinja index b054e0656..1e43a3ec7 100644 --- a/salt/kafka/config.map.jinja +++ b/salt/kafka/config.map.jinja @@ -6,8 +6,8 @@ {% from 'vars/globals.map.jinja' import GLOBALS %} {% set KAFKA_NODES_PILLAR = salt['pillar.get']('kafka:nodes') %} -{% set KAFKA_PASSWORD = salt['pillar.get']('kafka:password') %} -{% set KAFKA_TRUSTPASS = salt['pillar.get']('kafka:trustpass') %} +{% set KAFKA_PASSWORD = salt['pillar.get']('kafka:config:password') %} +{% set KAFKA_TRUSTPASS = salt['pillar.get']('kafka:config:trustpass') %} {# Create list of KRaft controllers #} {% set controllers = [] %} diff --git a/salt/kafka/ssl.sls b/salt/kafka/ssl.sls index 50a01b22c..8604d4e7e 100644 --- a/salt/kafka/ssl.sls +++ b/salt/kafka/ssl.sls @@ -6,7 +6,7 @@ {% from 'allowed_states.map.jinja' import allowed_states %} {% if sls.split('.')[0] in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} -{% set kafka_password = salt['pillar.get']('kafka:password') %} +{% set kafka_password = salt['pillar.get']('kafka:config:password') %} include: - ca.dirs diff --git a/salt/kafka/tools/sbin_jinja/so-kafka-trust b/salt/kafka/tools/sbin_jinja/so-kafka-trust index 8d404cb9a..55ba9612e 100644 --- a/salt/kafka/tools/sbin_jinja/so-kafka-trust +++ b/salt/kafka/tools/sbin_jinja/so-kafka-trust @@ -4,7 +4,7 @@ # or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{% set TRUSTPASS = salt['pillar.get']('kafka:trustpass') %} +{% set TRUSTPASS = salt['pillar.get']('kafka:config:trustpass') %} if [ ! -f /opt/so/saltstack/local/salt/kafka/files/kafka-truststore ]; then docker run -v /etc/pki/ca.crt:/etc/pki/ca.crt --name so-kafkatrust --user root --entrypoint /opt/java/openjdk/bin/keytool {{ GLOBALS.registry_host }}:5000/{{ GLOBALS.image_repo }}/so-kafka:{{ GLOBALS.so_version }} -import -file /etc/pki/ca.crt -alias SOS -keystore /etc/pki/kafka-truststore -storepass {{ TRUSTPASS }} -storetype jks -noprompt diff --git a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja index 00dd6d530..7478375b0 100644 --- a/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja +++ b/salt/logstash/pipelines/config/so/0800_input_kafka.conf.jinja @@ -1,5 +1,5 @@ -{%- set kafka_password = salt['pillar.get']('kafka:password') %} -{%- set kafka_trustpass = salt['pillar.get']('kafka:trustpass') %} +{%- set kafka_password = salt['pillar.get']('kafka:config:password') %} +{%- set kafka_trustpass = salt['pillar.get']('kafka:config:trustpass') %} {%- set kafka_brokers = salt['pillar.get']('kafka:nodes', {}) %} {%- set brokers = [] %} diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 019f29ebb..0ace4a5b2 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -674,7 +674,13 @@ up_to_2.4.80() { up_to_2.4.90() { kafkatrust=$(get_random_value) - echo ' trustpass: '$kafkatrust >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls + # rearranging the kafka pillar to reduce clutter in SOC UI + kafkasavedpass=$(so-yaml.py get /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password) + kafkatrimpass=$(echo $kafkasavedpass | awk '{print $1}') + so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password + echo ' config:' >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls + echo ' password: '$kafkatrimpass >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls + echo ' trustpass: '$kafkatrust >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls INSTALLEDVERSION=2.4.90 diff --git a/setup/so-functions b/setup/so-functions index aa9eb1909..02467117e 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1187,8 +1187,9 @@ kafka_pillar() { printf '%s\n'\ "kafka:"\ " cluster_id: $KAFKACLUSTERID"\ - " password: $KAFKAPASS"\ - " trustpass: $KAFKATRUST" > $kafka_pillar_file + " config:"\ + " password: $KAFKAPASS"\ + " trustpass: $KAFKATRUST" > $kafka_pillar_file } logrotate_pillar() { From 7ea8d5efd01bd26cc91dbef41e4d287697a4b4ec Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 12 Jul 2024 14:44:10 -0400 Subject: [PATCH 55/80] Remove redis input pipeline from searchnodes when global pipeline is Kafka Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/logstash/map.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/logstash/map.jinja b/salt/logstash/map.jinja index 2c3e02864..8fc3291e5 100644 --- a/salt/logstash/map.jinja +++ b/salt/logstash/map.jinja @@ -33,6 +33,7 @@ {# Append Kafka input pipeline when Kafka is enabled #} {% if GLOBALS.pipeline == 'KAFKA' %} +{% do LOGSTASH_MERGED.defined_pipelines.search.remove('so/0900_input_redis.conf.jinja') %} {% do LOGSTASH_MERGED.defined_pipelines.search.append('so/0800_input_kafka.conf.jinja') %} {% do LOGSTASH_MERGED.defined_pipelines.manager.append('so/0800_input_kafka.conf.jinja') %} {# Disable logstash on manager & receiver nodes unless it has an override configured #} From 5d322ebc0b4a6d17521503c05c824aa574619848 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 12 Jul 2024 14:45:11 -0400 Subject: [PATCH 56/80] Allow searchnodes to run kafka.ssl state for kafka-logstash cert generation Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/kafka/ssl.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/kafka/ssl.sls b/salt/kafka/ssl.sls index 8604d4e7e..04b6b4ba7 100644 --- a/salt/kafka/ssl.sls +++ b/salt/kafka/ssl.sls @@ -4,7 +4,7 @@ # Elastic License 2.0. {% from 'allowed_states.map.jinja' import allowed_states %} -{% if sls.split('.')[0] in allowed_states %} +{% if sls.split('.')[0] in allowed_states or sls in allowed_states %} {% from 'vars/globals.map.jinja' import GLOBALS %} {% set kafka_password = salt['pillar.get']('kafka:config:password') %} From fe1824aedd776bf9a286f67c41b6c68118538031 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 15 Jul 2024 11:28:59 -0400 Subject: [PATCH 57/80] Revert "Elastic 8.14.2" --- salt/common/tools/sbin/so-common | 2 +- .../elastic-defend-endpoints.json | 2 +- .../endpoints-initial/windows-defender.json | 2 +- .../tools/sbin_jinja/so-elastic-fleet-setup | 21 +- salt/elasticsearch/defaults.yaml | 2660 +++++++---------- salt/elasticsearch/soc_elasticsearch.yaml | 52 - .../so-elasticsearch-templates-load | 4 +- salt/kibana/files/config_saved_objects.ndjson | 2 +- .../tools/sbin_jinja/so-kibana-config-load | 2 +- 9 files changed, 1103 insertions(+), 1644 deletions(-) diff --git a/salt/common/tools/sbin/so-common b/salt/common/tools/sbin/so-common index 8a6effa5c..05c47a6c1 100755 --- a/salt/common/tools/sbin/so-common +++ b/salt/common/tools/sbin/so-common @@ -8,7 +8,7 @@ # Elastic agent is not managed by salt. Because of this we must store this base information in a # script that accompanies the soup system. Since so-common is one of those special soup files, # and since this same logic is required during installation, it's included in this file. -ELASTIC_AGENT_TARBALL_VERSION="8.14.2" +ELASTIC_AGENT_TARBALL_VERSION="8.10.4" ELASTIC_AGENT_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" ELASTIC_AGENT_MD5_URL="https://repo.securityonion.net/file/so-repo/prod/2.4/elasticagent/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.md5" ELASTIC_AGENT_FILE="/nsm/elastic-fleet/artifacts/elastic-agent_SO-$ELASTIC_AGENT_TARBALL_VERSION.tar.gz" diff --git a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json index 15f08a151..de35f803b 100644 --- a/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json +++ b/salt/elasticfleet/files/integrations/elastic-defend/elastic-defend-endpoints.json @@ -5,7 +5,7 @@ "package": { "name": "endpoint", "title": "Elastic Defend", - "version": "8.14.0" + "version": "8.10.2" }, "enabled": true, "policy_id": "endpoints-initial", diff --git a/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json index ab7e0783f..ac4394e62 100644 --- a/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json +++ b/salt/elasticfleet/files/integrations/endpoints-initial/windows-defender.json @@ -11,7 +11,7 @@ "winlogs-winlog": { "enabled": true, "streams": { - "winlog.winlogs": { + "winlog.winlog": { "enabled": true, "vars": { "channel": "Microsoft-Windows-Windows Defender/Operational", diff --git a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup index 7e497f6f5..0748557fd 100755 --- a/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup +++ b/salt/elasticfleet/tools/sbin_jinja/so-elastic-fleet-setup @@ -53,8 +53,7 @@ fi printf "\n### Create ES Token ###\n" ESTOKEN=$(curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/service_tokens" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' | jq -r .value) -### Create Outputs, Fleet Policy and Fleet URLs ### -# Create the Manager Elasticsearch Output first and set it as the default output +### Create Outputs & Fleet URLs ### printf "\nAdd Manager Elasticsearch Output...\n" ESCACRT=$(openssl x509 -in $INTCA) JSON_STRING=$( jq -n \ @@ -63,13 +62,7 @@ JSON_STRING=$( jq -n \ curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" -# Create the Manager Fleet Server Host Agent Policy -# This has to be done while the Elasticsearch Output is set to the default Output -printf "Create Manager Fleet Server Policy...\n" -elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120" - -# Now we can create the Logstash Output and set it to to be the default Output -printf "\n\nCreate Logstash Output Config if node is not an Import or Eval install\n" +printf "\nCreate Logstash Output Config if node is not an Import or Eval install\n" {% if grains.role not in ['so-import', 'so-eval'] %} LOGSTASHCRT=$(openssl x509 -in /etc/pki/elasticfleet-logstash.crt) LOGSTASHKEY=$(openssl rsa -in /etc/pki/elasticfleet-logstash.key) @@ -108,6 +101,16 @@ printf "\n\n" # Load Elasticsearch templates /usr/sbin/so-elasticsearch-templates-load +# Manager Fleet Server Host +elastic_fleet_policy_create "FleetServer_{{ GLOBALS.hostname }}" "Fleet Server - {{ GLOBALS.hostname }}" "true" "120" + +#Temp Fixup for ES Output bug +JSON_STRING=$( jq -n \ + --arg NAME "FleetServer_{{ GLOBALS.hostname }}" \ + '{"name": $NAME,"description": $NAME,"namespace":"default","monitoring_enabled":["logs"],"inactivity_timeout":120,"data_output_id":"so-manager_elasticsearch"}' + ) +curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/agent_policies/FleetServer_{{ GLOBALS.hostname }}" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" + # Initial Endpoints Policy elastic_fleet_policy_create "endpoints-initial" "Initial Endpoint Policy" "false" "1209600" diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index e1a2d192f..36d673d70 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -56,6 +56,87 @@ elasticsearch: enabled: true key: /usr/share/elasticsearch/config/elasticsearch.key verification_mode: none + pipelines: + custom001: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom001 + - pipeline: + name: common + custom002: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom002 + - pipeline: + name: common + custom003: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom003 + - pipeline: + name: common + custom004: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom004 + - pipeline: + name: common + custom005: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom005 + - pipeline: + name: common + custom006: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom006 + - pipeline: + name: common + custom007: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom007 + - pipeline: + name: common + custom008: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom008 + - pipeline: + name: common + custom009: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom009 + - pipeline: + name: common + custom010: + description: Custom Pipeline + processors: + - set: + field: tags + value: custom010 + - pipeline: + name: common index_settings: global_overrides: index_template: @@ -89,13 +170,84 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-items: + index_sorting: false + index_template: + composed_of: + - so-items-mappings + index_patterns: + - .items-default-** + priority: 500 + template: + mappings: + date_detection: false + settings: + index: + lifecycle: + name: so-items-logs + rollover_alias: ".items-default" + routing: + allocation: + include: + _tier_preference: "data_content" + mapping: + total_fields: + limit: 10000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: + rollover: + max_size: 50gb + min_age: 0ms + so-lists: + index_sorting: false + index_template: + composed_of: + - so-lists-mappings + index_patterns: + - .lists-default-** + priority: 500 + template: + mappings: + date_detection: false + settings: + index: + lifecycle: + name: so-lists-logs + rollover_alias: ".lists-default" + routing: + allocation: + include: + _tier_preference: "data_content" + mapping: + total_fields: + limit: 10000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + hot: + actions: + rollover: + max_size: 50gb + min_age: 0ms so-case: index_sorting: false index_template: composed_of: - case-mappings - case-settings - ignore_missing_component_templates: [] index_patterns: - so-case* priority: 500 @@ -119,6 +271,142 @@ elasticsearch: sort: field: '@timestamp' order: desc + so-detection: + index_sorting: false + index_template: + composed_of: + - detection-mappings + - detection-settings + index_patterns: + - so-detection* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + mapping: + total_fields: + limit: 1500 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + so-logs-soc: + close: 30 + delete: 365 + index_sorting: false + index_template: + composed_of: + - agent-mappings + - dtc-agent-mappings + - base-mappings + - dtc-base-mappings + - client-mappings + - dtc-client-mappings + - container-mappings + - destination-mappings + - dtc-destination-mappings + - pb-override-destination-mappings + - dll-mappings + - dns-mappings + - dtc-dns-mappings + - ecs-mappings + - dtc-ecs-mappings + - error-mappings + - event-mappings + - dtc-event-mappings + - file-mappings + - dtc-file-mappings + - group-mappings + - host-mappings + - dtc-host-mappings + - http-mappings + - dtc-http-mappings + - log-mappings + - network-mappings + - dtc-network-mappings + - observer-mappings + - dtc-observer-mappings + - organization-mappings + - package-mappings + - process-mappings + - dtc-process-mappings + - related-mappings + - rule-mappings + - dtc-rule-mappings + - server-mappings + - service-mappings + - dtc-service-mappings + - source-mappings + - dtc-source-mappings + - pb-override-source-mappings + - threat-mappings + - tls-mappings + - url-mappings + - user_agent-mappings + - dtc-user_agent-mappings + - common-settings + - common-dynamic-mappings + data_stream: {} + index_patterns: + - logs-soc-so* + priority: 500 + template: + mappings: + date_detection: false + dynamic_templates: + - strings_as_keyword: + mapping: + ignore_above: 1024 + type: keyword + match_mapping_type: string + settings: + index: + lifecycle: + name: so-soc-logs + mapping: + total_fields: + limit: 5000 + number_of_replicas: 0 + number_of_shards: 1 + refresh_interval: 30s + sort: + field: '@timestamp' + order: desc + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + warm: 7 so-common: close: 30 delete: 365 @@ -185,7 +473,6 @@ elasticsearch: - common-dynamic-mappings - winlog-mappings data_stream: {} - ignore_missing_component_templates: [] index_patterns: - logs-*-so* priority: 1 @@ -236,36 +523,6 @@ elasticsearch: priority: 50 min_age: 30d warm: 7 - so-detection: - index_sorting: false - index_template: - composed_of: - - detection-mappings - - detection-settings - ignore_missing_component_templates: [] - index_patterns: - - so-detection* - priority: 500 - template: - mappings: - date_detection: false - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - settings: - index: - mapping: - total_fields: - limit: 1500 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc so-endgame: index_sorting: false index_template: @@ -328,7 +585,6 @@ elasticsearch: - common-settings - common-dynamic-mappings - winlog-mappings - ignore_missing_component_templates: [] index_patterns: - endgame* priority: 500 @@ -434,7 +690,6 @@ elasticsearch: - dtc-user_agent-mappings - common-settings - common-dynamic-mappings - ignore_missing_component_templates: [] index_patterns: - so-idh-* priority: 500 @@ -547,7 +802,6 @@ elasticsearch: - common-dynamic-mappings - winlog-mappings data_stream: {} - ignore_missing_component_templates: [] index_patterns: - logs-import-so* priority: 500 @@ -598,91 +852,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-items: - index_sorting: false - index_template: - composed_of: - - so-items-mappings - ignore_missing_component_templates: [] - index_patterns: - - .items-default-** - priority: 500 - template: - mappings: - date_detection: false - settings: - index: - lifecycle: - name: so-items-logs - rollover_alias: .items-default - mapping: - total_fields: - limit: 10000 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - routing: - allocation: - include: - _tier_preference: data_content - sort: - field: '@timestamp' - order: desc - policy: - phases: - hot: - actions: - rollover: - max_size: 50gb - min_age: 0ms - so-kismet: - index_sorting: false - index_template: - composed_of: - - kismet-mappings - - source-mappings - - client-mappings - - device-mappings - - network-mappings - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: [] - index_patterns: - - logs-kismet-so* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-kismet-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d so-kratos: close: 30 delete: 365 @@ -742,7 +911,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: [] index_patterns: - logs-kratos-so* priority: 500 @@ -793,43 +961,6 @@ elasticsearch: priority: 50 min_age: 30d warm: 7 - so-lists: - index_sorting: false - index_template: - composed_of: - - so-lists-mappings - ignore_missing_component_templates: [] - index_patterns: - - .lists-default-** - priority: 500 - template: - mappings: - date_detection: false - settings: - index: - lifecycle: - name: so-lists-logs - rollover_alias: .lists-default - mapping: - total_fields: - limit: 10000 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - routing: - allocation: - include: - _tier_preference: data_content - sort: - field: '@timestamp' - order: desc - policy: - phases: - hot: - actions: - rollover: - max_size: 50gb - min_age: 0ms so-logs: index_sorting: false index_template: @@ -842,7 +973,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: [] index_patterns: - logs-*-* priority: 225 @@ -904,8 +1034,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-1password.item_usages@custom index_patterns: - logs-1password.item_usages-* priority: 501 @@ -950,8 +1078,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-1password.signin_attempts@custom index_patterns: - logs-1password.signin_attempts-* priority: 501 @@ -996,8 +1122,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-apache.access@custom index_patterns: - logs-apache.access-* priority: 501 @@ -1042,8 +1166,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-apache.error@custom index_patterns: - logs-apache.error-* priority: 501 @@ -1088,8 +1210,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-auditd.log@custom index_patterns: - logs-auditd.log-* priority: 501 @@ -1134,8 +1254,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-auth0.logs@custom index_patterns: - logs-auth0.logs-* priority: 501 @@ -1170,27 +1288,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_cloudfront_logs: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-aws.cloudfront_logs@package - - logs-aws.cloudfront_logs@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-aws.cloudfront_logs@custom index_patterns: - - logs-aws.cloudfront_logs-* - priority: 501 + - "logs-aws.cloudfront_logs-*" template: settings: index: lifecycle: name: so-logs-aws.cloudfront_logs-logs number_of_replicas: 0 + composed_of: + - "logs-aws.cloudfront_logs@package" + - "logs-aws.cloudfront_logs@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -1226,8 +1342,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.cloudtrail@custom index_patterns: - logs-aws.cloudtrail-* priority: 501 @@ -1272,8 +1386,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.cloudwatch_logs@custom index_patterns: - logs-aws.cloudwatch_logs-* priority: 501 @@ -1318,8 +1430,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.ec2_logs@custom index_patterns: - logs-aws.ec2_logs-* priority: 501 @@ -1364,8 +1474,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.elb_logs@custom index_patterns: - logs-aws.elb_logs-* priority: 501 @@ -1410,8 +1518,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.firewall_logs@custom index_patterns: - logs-aws.firewall_logs-* priority: 501 @@ -1446,27 +1552,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_guardduty: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-aws.guardduty@package - - logs-aws.guardduty@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-aws.guardduty@custom index_patterns: - - logs-aws.guardduty-* - priority: 501 + - "logs-aws.guardduty-*" template: settings: index: lifecycle: name: so-logs-aws.guardduty-logs number_of_replicas: 0 + composed_of: + - "logs-aws.guardduty@package" + - "logs-aws.guardduty@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -1492,27 +1596,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_inspector: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-aws.inspector@package - - logs-aws.inspector@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-aws.inspector@custom index_patterns: - - logs-aws.inspector-* - priority: 501 + - "logs-aws.inspector-*" template: settings: index: lifecycle: name: so-logs-aws.inspector-logs number_of_replicas: 0 + composed_of: + - "logs-aws.inspector@package" + - "logs-aws.inspector@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -1548,8 +1650,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.route53_public_logs@custom index_patterns: - logs-aws.route53_public_logs-* priority: 501 @@ -1594,8 +1694,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.route53_resolver_logs@custom index_patterns: - logs-aws.route53_resolver_logs-* priority: 501 @@ -1640,8 +1738,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.s3access@custom index_patterns: - logs-aws.s3access-* priority: 501 @@ -1676,27 +1772,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_securityhub_findings: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-aws.securityhub_findings@package - - logs-aws.securityhub_findings@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-aws.securityhub_findings@custom index_patterns: - - logs-aws.securityhub_findings-* - priority: 501 + - "logs-aws.securityhub_findings-*" template: settings: index: lifecycle: name: so-logs-aws.securityhub_findings-logs number_of_replicas: 0 + composed_of: + - "logs-aws.securityhub_findings@package" + - "logs-aws.securityhub_findings@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -1722,27 +1816,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-aws_x_securityhub_insights: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-aws.securityhub_insights@package - - logs-aws.securityhub_insights@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-aws.securityhub_insights@custom index_patterns: - - logs-aws.securityhub_insights-* - priority: 501 + - "logs-aws.securityhub_insights-*" template: settings: index: lifecycle: name: so-logs-aws.securityhub_insights-logs number_of_replicas: 0 + composed_of: + - "logs-aws.securityhub_insights@package" + - "logs-aws.securityhub_insights@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -1778,8 +1870,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.vpcflow@custom index_patterns: - logs-aws.vpcflow-* priority: 501 @@ -1824,8 +1914,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-aws.waf@custom index_patterns: - logs-aws.waf-* priority: 501 @@ -1870,8 +1958,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.activitylogs@custom index_patterns: - logs-azure.activitylogs-* priority: 501 @@ -1916,8 +2002,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.application_gateway@custom index_patterns: - logs-azure.application_gateway-* priority: 501 @@ -1962,8 +2046,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.auditlogs@custom index_patterns: - logs-azure.auditlogs-* priority: 501 @@ -2008,8 +2090,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.eventhub@custom index_patterns: - logs-azure.eventhub-* priority: 501 @@ -2054,8 +2134,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.firewall_logs@custom index_patterns: - logs-azure.firewall_logs-* priority: 501 @@ -2100,8 +2178,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.identity_protection@custom index_patterns: - logs-azure.identity_protection-* priority: 501 @@ -2146,8 +2222,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.platformlogs@custom index_patterns: - logs-azure.platformlogs-* priority: 501 @@ -2192,8 +2266,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.provisioning@custom index_patterns: - logs-azure.provisioning-* priority: 501 @@ -2238,8 +2310,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.signinlogs@custom index_patterns: - logs-azure.signinlogs-* priority: 501 @@ -2284,8 +2354,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-azure.springcloudlogs@custom index_patterns: - logs-azure.springcloudlogs-* priority: 501 @@ -2330,8 +2398,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-barracuda.waf@custom index_patterns: - logs-barracuda.waf-* priority: 501 @@ -2376,8 +2442,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-carbonblack_edr.log@custom index_patterns: - logs-carbonblack_edr.log-* priority: 501 @@ -2412,27 +2476,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cef_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-cef.log@package - - logs-cef.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-cef.log@custom index_patterns: - - logs-cef.log-* - priority: 501 + - "logs-cef.log-*" template: settings: index: lifecycle: name: so-logs-cef.log-logs number_of_replicas: 0 + composed_of: + - "logs-cef.log@package" + - "logs-cef.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -2458,27 +2520,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-checkpoint_x_firewall: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-checkpoint.firewall@package - - logs-checkpoint.firewall@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-checkpoint.firewall@custom index_patterns: - - logs-checkpoint.firewall-* - priority: 501 + - "logs-checkpoint.firewall-*" template: settings: index: lifecycle: name: so-logs-checkpoint.firewall-logs number_of_replicas: 0 + composed_of: + - "logs-checkpoint.firewall@package" + - "logs-checkpoint.firewall@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -2514,8 +2574,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_asa.log@custom index_patterns: - logs-cisco_asa.log-* priority: 501 @@ -2560,8 +2618,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_duo.admin@custom index_patterns: - logs-cisco_duo.admin-* priority: 501 @@ -2606,8 +2662,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_duo.auth@custom index_patterns: - logs-cisco_duo.auth-* priority: 501 @@ -2652,8 +2706,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_duo.offline_enrollment@custom index_patterns: - logs-cisco_duo.offline_enrollment-* priority: 501 @@ -2698,8 +2750,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_duo.summary@custom index_patterns: - logs-cisco_duo.summary-* priority: 501 @@ -2744,8 +2794,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_duo.telephony@custom index_patterns: - logs-cisco_duo.telephony-* priority: 501 @@ -2780,27 +2828,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ftd_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-cisco_ftd.log@package - - logs-cisco_ftd.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-cisco_ftd.log@custom index_patterns: - - logs-cisco_ftd.log-* - priority: 501 + - "logs-cisco_ftd.log-*" template: settings: index: lifecycle: name: so-logs-cisco_ftd.log-logs number_of_replicas: 0 + composed_of: + - "logs-cisco_ftd.log@package" + - "logs-cisco_ftd.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -2826,27 +2872,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ios_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-cisco_ios.log@package - - logs-cisco_ios.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-cisco_ios.log@custom index_patterns: - - logs-cisco_ios.log-* - priority: 501 + - "logs-cisco_ios.log-*" template: settings: index: lifecycle: name: so-logs-cisco_ios.log-logs number_of_replicas: 0 + composed_of: + - "logs-cisco_ios.log@package" + - "logs-cisco_ios.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -2872,27 +2916,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-cisco_ise_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-cisco_ise.log@package - - logs-cisco_ise.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-cisco_ise.log@custom index_patterns: - - logs-cisco_ise.log-* - priority: 501 + - "logs-cisco_ise.log-*" template: settings: index: lifecycle: name: so-logs-cisco_ise.log-logs number_of_replicas: 0 + composed_of: + - "logs-cisco_ise.log@package" + - "logs-cisco_ise.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -2928,8 +2970,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_meraki.events@custom index_patterns: - logs-cisco_meraki.events-* priority: 501 @@ -2974,8 +3014,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_meraki.log@custom index_patterns: - logs-cisco_meraki.log-* priority: 501 @@ -3020,8 +3058,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cisco_umbrella.log@custom index_patterns: - logs-cisco_umbrella.log-* priority: 501 @@ -3056,27 +3092,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_interface: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-citrix_adc.interface@package - - logs-citrix_adc.interface@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-citrix_adc.interface@custom index_patterns: - - logs-citrix_adc.interface-* - priority: 501 + - "logs-citrix_adc.interface-*" template: settings: index: lifecycle: name: so-logs-citrix_adc.interface-logs number_of_replicas: 0 + composed_of: + - "logs-citrix_adc.interface@package" + - "logs-citrix_adc.interface@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -3102,27 +3136,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_lbvserver: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-citrix_adc.lbvserver@package - - logs-citrix_adc.lbvserver@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-citrix_adc.lbvserver@custom index_patterns: - - logs-citrix_adc.lbvserver-* - priority: 501 + - "logs-citrix_adc.lbvserver-*" template: settings: index: lifecycle: name: so-logs-citrix_adc.lbvserver-logs number_of_replicas: 0 + composed_of: + - "logs-citrix_adc.lbvserver@package" + - "logs-citrix_adc.lbvserver@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -3148,27 +3180,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_service: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-citrix_adc.service@package - - logs-citrix_adc.service@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-citrix_adc.service@custom index_patterns: - - logs-citrix_adc.service-* - priority: 501 + - "logs-citrix_adc.service-*" template: settings: index: lifecycle: name: so-logs-citrix_adc.service-logs number_of_replicas: 0 + composed_of: + - "logs-citrix_adc.service@package" + - "logs-citrix_adc.service@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -3194,27 +3224,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_system: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-citrix_adc.system@package - - logs-citrix_adc.system@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-citrix_adc.system@custom index_patterns: - - logs-citrix_adc.system-* - priority: 501 + - "logs-citrix_adc.system-*" template: settings: index: lifecycle: name: so-logs-citrix_adc.system-logs number_of_replicas: 0 + composed_of: + - "logs-citrix_adc.system@package" + - "logs-citrix_adc.system@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -3240,27 +3268,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_adc_x_vpn: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-citrix_adc.vpn@package - - logs-citrix_adc.vpn@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-citrix_adc.vpn@custom index_patterns: - - logs-citrix_adc.vpn-* - priority: 501 + - "logs-citrix_adc.vpn-*" template: settings: index: lifecycle: name: so-logs-citrix_adc.vpn-logs number_of_replicas: 0 + composed_of: + - "logs-citrix_adc.vpn@package" + - "logs-citrix_adc.vpn@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -3286,27 +3312,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-citrix_waf_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-citrix_waf.log@package - - logs-citrix_waf.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-citrix_waf.log@custom index_patterns: - - logs-citrix_waf.log-* - priority: 501 + - "logs-citrix_waf.log-*" template: settings: index: lifecycle: name: so-logs-citrix_waf.log-logs number_of_replicas: 0 + composed_of: + - "logs-citrix_waf.log@package" + - "logs-citrix_waf.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -3342,8 +3366,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cloudflare.audit@custom index_patterns: - logs-cloudflare.audit-* priority: 501 @@ -3388,8 +3410,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-cloudflare.logpull@custom index_patterns: - logs-cloudflare.logpull-* priority: 501 @@ -3434,8 +3454,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-crowdstrike.falcon@custom index_patterns: - logs-crowdstrike.falcon-* priority: 501 @@ -3480,8 +3498,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-crowdstrike.fdr@custom index_patterns: - logs-crowdstrike.fdr-* priority: 501 @@ -3526,8 +3542,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-darktrace.ai_analyst_alert@custom index_patterns: - logs-darktrace.ai_analyst_alert-* priority: 501 @@ -3572,8 +3586,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-darktrace.model_breach_alert@custom index_patterns: - logs-darktrace.model_breach_alert-* priority: 501 @@ -3618,8 +3630,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-darktrace.system_status_alert@custom index_patterns: - logs-darktrace.system_status_alert-* priority: 501 @@ -3665,7 +3675,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: [] index_patterns: - logs-detections.alerts-* priority: 501 @@ -3728,8 +3737,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent@custom index_patterns: - logs-elastic_agent-* priority: 501 @@ -3791,8 +3798,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.apm_server@custom index_patterns: - logs-elastic_agent.apm_server-* priority: 501 @@ -3854,8 +3859,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.auditbeat@custom index_patterns: - logs-elastic_agent.auditbeat-* priority: 501 @@ -3914,8 +3917,6 @@ elasticsearch: - logs-elastic_agent.cloudbeat@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 - ignore_missing_component_templates: - - logs-elastic_agent.cloudbeat@custom index_patterns: - logs-elastic_agent.cloudbeat-* priority: 501 @@ -3978,8 +3979,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.endpoint_security@custom index_patterns: - logs-elastic_agent.endpoint_security-* priority: 501 @@ -4036,8 +4035,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.filebeat@custom index_patterns: - logs-elastic_agent.filebeat-* priority: 501 @@ -4094,8 +4091,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.fleet_server@custom index_patterns: - logs-elastic_agent.fleet_server-* priority: 501 @@ -4145,8 +4140,6 @@ elasticsearch: - logs-elastic_agent.heartbeat@custom - so-fleet_globals-1 - so-fleet_agent_id_verification-1 - ignore_missing_component_templates: - - logs-elastic_agent.heartbeat@custom index_patterns: - logs-elastic_agent.heartbeat-* priority: 501 @@ -4209,8 +4202,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.metricbeat@custom index_patterns: - logs-elastic_agent.metricbeat-* priority: 501 @@ -4267,8 +4258,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.osquerybeat@custom index_patterns: - logs-elastic_agent.osquerybeat-* priority: 501 @@ -4324,8 +4313,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-elastic_agent.packetbeat@custom index_patterns: - logs-elastic_agent.packetbeat-* priority: 501 @@ -4388,8 +4375,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.alerts@custom index_patterns: - logs-endpoint.alerts-* priority: 501 @@ -4446,8 +4431,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.diagnostic.collection@custom index_patterns: - .logs-endpoint.diagnostic.collection-* priority: 501 @@ -4504,8 +4487,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.events.api@custom index_patterns: - logs-endpoint.events.api-* priority: 501 @@ -4562,8 +4543,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.events.file@custom index_patterns: - logs-endpoint.events.file-* priority: 501 @@ -4620,8 +4599,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.events.library@custom index_patterns: - logs-endpoint.events.library-* priority: 501 @@ -4678,8 +4655,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.events.network@custom index_patterns: - logs-endpoint.events.network-* priority: 501 @@ -4736,8 +4711,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.events.process@custom index_patterns: - logs-endpoint.events.process-* priority: 501 @@ -4794,8 +4767,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.events.registry@custom index_patterns: - logs-endpoint.events.registry-* priority: 501 @@ -4852,8 +4823,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-endpoint.events.security@custom index_patterns: - logs-endpoint.events.security-* priority: 501 @@ -4909,8 +4878,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-f5_bigip.log@custom index_patterns: - logs-f5_bigip.log-* priority: 501 @@ -4955,8 +4922,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-fim.event@custom index_patterns: - logs-fim.event-* priority: 501 @@ -5001,8 +4966,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-fireeye.nx@custom index_patterns: - logs-fireeye.nx-* priority: 501 @@ -5047,8 +5010,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-fortinet_fortigate.log@custom index_patterns: - logs-fortinet_fortigate.log-* priority: 501 @@ -5093,8 +5054,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-fortinet.clientendpoint@custom index_patterns: - logs-fortinet.clientendpoint-* priority: 501 @@ -5139,8 +5098,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-fortinet.firewall@custom index_patterns: - logs-fortinet.firewall-* priority: 501 @@ -5185,8 +5142,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-fortinet.fortimail@custom index_patterns: - logs-fortinet.fortimail-* priority: 501 @@ -5231,8 +5186,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-fortinet.fortimanager@custom index_patterns: - logs-fortinet.fortimanager-* priority: 501 @@ -5277,8 +5230,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-gcp.audit@custom index_patterns: - logs-gcp.audit-* priority: 501 @@ -5323,8 +5274,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-gcp.dns@custom index_patterns: - logs-gcp.dns-* priority: 501 @@ -5369,8 +5318,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-gcp.firewall@custom index_patterns: - logs-gcp.firewall-* priority: 501 @@ -5415,8 +5362,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-gcp.loadbalancing_logs@custom index_patterns: - logs-gcp.loadbalancing_logs-* priority: 501 @@ -5461,8 +5406,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-gcp.vpcflow@custom index_patterns: - logs-gcp.vpcflow-* priority: 501 @@ -5507,8 +5450,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-github.audit@custom index_patterns: - logs-github.audit-* priority: 501 @@ -5553,8 +5494,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-github.code_scanning@custom index_patterns: - logs-github.code_scanning-* priority: 501 @@ -5599,8 +5538,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-github.dependabot@custom index_patterns: - logs-github.dependabot-* priority: 501 @@ -5645,8 +5582,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-github.issues@custom index_patterns: - logs-github.issues-* priority: 501 @@ -5691,8 +5626,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-github.secret_scanning@custom index_patterns: - logs-github.secret_scanning-* priority: 501 @@ -5737,8 +5670,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.access_transparency@custom index_patterns: - logs-google_workspace.access_transparency-* priority: 501 @@ -5783,8 +5714,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.admin@custom index_patterns: - logs-google_workspace.admin-* priority: 501 @@ -5829,8 +5758,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.alert@custom index_patterns: - logs-google_workspace.alert-* priority: 501 @@ -5875,8 +5802,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.context_aware_access@custom index_patterns: - logs-google_workspace.context_aware_access-* priority: 501 @@ -5921,8 +5846,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.device@custom index_patterns: - logs-google_workspace.device-* priority: 501 @@ -5967,8 +5890,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.drive@custom index_patterns: - logs-google_workspace.drive-* priority: 501 @@ -6013,8 +5934,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.gcp@custom index_patterns: - logs-google_workspace.gcp-* priority: 501 @@ -6059,8 +5978,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.group_enterprise@custom index_patterns: - logs-google_workspace.group_enterprise-* priority: 501 @@ -6105,8 +6022,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.groups@custom index_patterns: - logs-google_workspace.groups-* priority: 501 @@ -6151,8 +6066,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.login@custom index_patterns: - logs-google_workspace.login-* priority: 501 @@ -6197,8 +6110,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.rules@custom index_patterns: - logs-google_workspace.rules-* priority: 501 @@ -6243,8 +6154,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.saml@custom index_patterns: - logs-google_workspace.saml-* priority: 501 @@ -6289,8 +6198,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.token@custom index_patterns: - logs-google_workspace.token-* priority: 501 @@ -6335,8 +6242,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-google_workspace.user_accounts@custom index_patterns: - logs-google_workspace.user_accounts-* priority: 501 @@ -6381,9 +6286,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-http_endpoint.generic@package - - logs-http_endpoint.generic@custom index_patterns: - logs-http_endpoint.generic-* priority: 501 @@ -6428,8 +6330,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-httpjson.generic@custom index_patterns: - logs-httpjson.generic-* priority: 501 @@ -6464,27 +6364,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-iis_x_access: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-iis.access@package - - logs-iis.access@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-iis.access@custom index_patterns: - - logs-iis.access-* - priority: 501 + - "logs-iis.access-*" template: settings: index: lifecycle: name: so-logs-iis.access-logs number_of_replicas: 0 + composed_of: + - "logs-iis.access@package" + - "logs-iis.access@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -6510,27 +6408,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-iis_x_error: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-iis.error@package - - logs-iis.error@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-iis.error@custom index_patterns: - - logs-iis.error-* - priority: 501 + - "logs-iis.error-*" template: settings: index: lifecycle: name: so-logs-iis.error-logs number_of_replicas: 0 + composed_of: + - "logs-iis.error@package" + - "logs-iis.error@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -6566,8 +6462,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-juniper_srx.log@custom index_patterns: - logs-juniper_srx.log-* priority: 501 @@ -6612,8 +6506,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-juniper.junos@custom index_patterns: - logs-juniper.junos-* priority: 501 @@ -6658,8 +6550,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-juniper.netscreen@custom index_patterns: - logs-juniper.netscreen-* priority: 501 @@ -6704,8 +6594,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-juniper.srx@custom index_patterns: - logs-juniper.srx-* priority: 501 @@ -6750,8 +6638,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-kafka_log.generic@custom index_patterns: - logs-kafka_log.generic-* priority: 501 @@ -6796,8 +6682,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-lastpass.detailed_shared_folder@custom index_patterns: - logs-lastpass.detailed_shared_folder-* priority: 501 @@ -6842,8 +6726,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-lastpass.event_report@custom index_patterns: - logs-lastpass.event_report-* priority: 501 @@ -6888,8 +6770,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-lastpass.user@custom index_patterns: - logs-lastpass.user-* priority: 501 @@ -6934,8 +6814,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-m365_defender.event@custom index_patterns: - logs-m365_defender.event-* priority: 501 @@ -6980,8 +6858,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-m365_defender.incident@custom index_patterns: - logs-m365_defender.incident-* priority: 501 @@ -7026,8 +6902,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-m365_defender.log@custom index_patterns: - logs-m365_defender.log-* priority: 501 @@ -7072,8 +6946,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-microsoft_defender_endpoint.log@custom index_patterns: - logs-microsoft_defender_endpoint.log-* priority: 501 @@ -7118,8 +6990,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-microsoft_dhcp.log@custom index_patterns: - logs-microsoft_dhcp.log-* priority: 501 @@ -7154,27 +7024,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-microsoft_sqlserver_x_audit: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-microsoft_sqlserver.audit@package - - logs-microsoft_sqlserver.audit@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-microsoft_sqlserver.audit@custom index_patterns: - - logs-microsoft_sqlserver.audit-* - priority: 501 + - "logs-microsoft_sqlserver.audit-*" template: settings: index: lifecycle: name: so-logs-microsoft_sqlserver.audit-logs number_of_replicas: 0 + composed_of: + - "logs-microsoft_sqlserver.audit@package" + - "logs-microsoft_sqlserver.audit@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -7200,27 +7068,113 @@ elasticsearch: priority: 50 min_age: 30d so-logs-microsoft_sqlserver_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-microsoft_sqlserver.log@package - - logs-microsoft_sqlserver.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-microsoft_sqlserver.log@custom index_patterns: - - logs-microsoft_sqlserver.log-* - priority: 501 + - "logs-microsoft_sqlserver.log-*" template: settings: index: lifecycle: name: so-logs-microsoft_sqlserver.log-logs number_of_replicas: 0 + composed_of: + - "logs-microsoft_sqlserver.log@package" + - "logs-microsoft_sqlserver.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-mysql_x_error: + index_sorting: False + index_template: + index_patterns: + - "logs-mysql.error-*" + template: + settings: + index: + lifecycle: + name: so-logs-mysql.error-logs + number_of_replicas: 0 + composed_of: + - "logs-mysql.error@package" + - "logs-mysql.error@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-logs-mysql_x_slowlog: + index_sorting: False + index_template: + index_patterns: + - "logs-mysql.slowlog-*" + template: + settings: + index: + lifecycle: + name: so-logs-mysql.slowlog-logs + number_of_replicas: 0 + composed_of: + - "logs-mysql.slowlog@package" + - "logs-mysql.slowlog@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -7256,8 +7210,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.audit_events@custom index_patterns: - logs-mimecast.audit_events-* priority: 501 @@ -7302,8 +7254,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.dlp_logs@custom index_patterns: - logs-mimecast.dlp_logs-* priority: 501 @@ -7348,8 +7298,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.siem_logs@custom index_patterns: - logs-mimecast.siem_logs-* priority: 501 @@ -7394,8 +7342,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.threat_intel_malware_customer@custom index_patterns: - logs-mimecast.threat_intel_malware_customer-* priority: 501 @@ -7440,8 +7386,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.threat_intel_malware_grid@custom index_patterns: - logs-mimecast.threat_intel_malware_grid-* priority: 501 @@ -7486,8 +7430,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.ttp_ap_logs@custom index_patterns: - logs-mimecast.ttp_ap_logs-* priority: 501 @@ -7532,8 +7474,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.ttp_ip_logs@custom index_patterns: - logs-mimecast.ttp_ip_logs-* priority: 501 @@ -7578,8 +7518,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-mimecast.ttp_url_logs@custom index_patterns: - logs-mimecast.ttp_url_logs-* priority: 501 @@ -7613,98 +7551,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-logs-mysql_x_error: - index_sorting: false - index_template: - composed_of: - - logs-mysql.error@package - - logs-mysql.error@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-mysql.error@custom - index_patterns: - - logs-mysql.error-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-logs-mysql.error-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-logs-mysql_x_slowlog: - index_sorting: false - index_template: - composed_of: - - logs-mysql.slowlog@package - - logs-mysql.slowlog@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-mysql.slowlog@custom - index_patterns: - - logs-mysql.slowlog-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-logs-mysql.slowlog-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d so-logs-netflow_x_log: index_sorting: false index_template: @@ -7716,8 +7562,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-netflow.log@custom index_patterns: - logs-netflow.log-* priority: 501 @@ -7752,27 +7596,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-nginx_x_access: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-nginx.access@package - - logs-nginx.access@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-nginx.access@custom index_patterns: - - logs-nginx.access-* - priority: 501 + - "logs-nginx.access-*" template: settings: index: lifecycle: name: so-logs-nginx.access-logs number_of_replicas: 0 + composed_of: + - "logs-nginx.access@package" + - "logs-nginx.access@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -7798,27 +7640,69 @@ elasticsearch: priority: 50 min_age: 30d so-logs-nginx_x_error: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-nginx.error@package - - logs-nginx.error@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-nginx.error@custom index_patterns: - - logs-nginx.error-* - priority: 501 + - "logs-nginx.error-*" template: settings: index: lifecycle: name: so-logs-nginx.error-logs number_of_replicas: 0 + composed_of: + - "logs-nginx.error@package" + - "logs-nginx.error@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-nginx_x_stubstatus: + index_sorting: False + index_template: + index_patterns: + - "metrics-nginx.stubstatus-*" + template: + settings: + index: + lifecycle: + name: so-metrics-nginx.stubstatus-logs + number_of_replicas: 0 + composed_of: + - "metrics-nginx.stubstatus@package" + - "metrics-nginx.stubstatus@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -7854,8 +7738,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-o365.audit@custom index_patterns: - logs-o365.audit-* priority: 501 @@ -7900,8 +7782,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-okta.system@custom index_patterns: - logs-okta.system-* priority: 501 @@ -7945,7 +7825,6 @@ elasticsearch: name: elastic_agent composed_of: - logs-osquery_manager.action.responses - ignore_missing_component_templates: [] index_patterns: - .logs-osquery_manager.action.responses* priority: 501 @@ -7963,7 +7842,6 @@ elasticsearch: name: elastic_agent composed_of: - logs-osquery_manager.actions - ignore_missing_component_templates: [] index_patterns: - .logs-osquery_manager.actions* priority: 501 @@ -7982,8 +7860,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-panw.panos@custom index_patterns: - logs-panw.panos-* priority: 501 @@ -8028,8 +7904,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-pfsense.log@custom index_patterns: - logs-pfsense.log-* priority: 501 @@ -8064,27 +7938,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_clicks_blocked: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-proofpoint_tap.clicks_blocked@package - - logs-proofpoint_tap.clicks_blocked@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-proofpoint_tap.clicks_blocked@custom index_patterns: - - logs-proofpoint_tap.clicks_blocked-* - priority: 501 + - "logs-proofpoint_tap.clicks_blocked-*" template: settings: index: lifecycle: name: so-logs-proofpoint_tap.clicks_blocked-logs number_of_replicas: 0 + composed_of: + - "logs-proofpoint_tap.clicks_blocked@package" + - "logs-proofpoint_tap.clicks_blocked@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -8110,27 +7982,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_clicks_permitted: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-proofpoint_tap.clicks_permitted@package - - logs-proofpoint_tap.clicks_permitted@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-proofpoint_tap.clicks_permitted@custom index_patterns: - - logs-proofpoint_tap.clicks_permitted-* - priority: 501 + - "logs-proofpoint_tap.clicks_permitted-*" template: settings: index: lifecycle: name: so-logs-proofpoint_tap.clicks_permitted-logs number_of_replicas: 0 + composed_of: + - "logs-proofpoint_tap.clicks_permitted@package" + - "logs-proofpoint_tap.clicks_permitted@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -8156,27 +8026,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_message_blocked: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-proofpoint_tap.message_blocked@package - - logs-proofpoint_tap.message_blocked@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-proofpoint_tap.message_blocked@custom index_patterns: - - logs-proofpoint_tap.message_blocked-* - priority: 501 + - "logs-proofpoint_tap.message_blocked-*" template: settings: index: lifecycle: name: so-logs-proofpoint_tap.message_blocked-logs number_of_replicas: 0 + composed_of: + - "logs-proofpoint_tap.message_blocked@package" + - "logs-proofpoint_tap.message_blocked@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -8202,27 +8070,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-proofpoint_tap_x_message_delivered: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-proofpoint_tap.message_delivered@package - - logs-proofpoint_tap.message_delivered@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-proofpoint_tap.message_delivered@custom index_patterns: - - logs-proofpoint_tap.message_delivered-* - priority: 501 + - "logs-proofpoint_tap.message_delivered-*" template: settings: index: lifecycle: name: so-logs-proofpoint_tap.message_delivered-logs number_of_replicas: 0 + composed_of: + - "logs-proofpoint_tap.message_delivered@package" + - "logs-proofpoint_tap.message_delivered@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -8258,8 +8124,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-pulse_connect_secure.log@custom index_patterns: - logs-pulse_connect_secure.log-* priority: 501 @@ -8304,8 +8168,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sentinel_one.activity@custom index_patterns: - logs-sentinel_one.activity-* priority: 501 @@ -8350,8 +8212,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sentinel_one.agent@custom index_patterns: - logs-sentinel_one.agent-* priority: 501 @@ -8396,8 +8256,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sentinel_one.alert@custom index_patterns: - logs-sentinel_one.alert-* priority: 501 @@ -8442,8 +8300,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sentinel_one.group@custom index_patterns: - logs-sentinel_one.group-* priority: 501 @@ -8488,8 +8344,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sentinel_one.threat@custom index_patterns: - logs-sentinel_one.threat-* priority: 501 @@ -8524,27 +8378,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-snort_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-snort.log@package - - logs-snort.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-snort.log@custom index_patterns: - - logs-snort.log-* - priority: 501 + - "logs-snort.log-*" template: settings: index: lifecycle: name: so-logs-snort.log-logs number_of_replicas: 0 + composed_of: + - "logs-snort.log@package" + - "logs-snort.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -8580,8 +8432,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-snyk.audit@custom index_patterns: - logs-snyk.audit-* priority: 501 @@ -8626,8 +8476,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-snyk.vulnerabilities@custom index_patterns: - logs-snyk.vulnerabilities-* priority: 501 @@ -8661,114 +8509,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-logs-soc: - close: 30 - delete: 365 - index_sorting: false - index_template: - composed_of: - - agent-mappings - - dtc-agent-mappings - - base-mappings - - dtc-base-mappings - - client-mappings - - dtc-client-mappings - - container-mappings - - destination-mappings - - dtc-destination-mappings - - pb-override-destination-mappings - - dll-mappings - - dns-mappings - - dtc-dns-mappings - - ecs-mappings - - dtc-ecs-mappings - - error-mappings - - event-mappings - - dtc-event-mappings - - file-mappings - - dtc-file-mappings - - group-mappings - - host-mappings - - dtc-host-mappings - - http-mappings - - dtc-http-mappings - - log-mappings - - network-mappings - - dtc-network-mappings - - observer-mappings - - dtc-observer-mappings - - organization-mappings - - package-mappings - - process-mappings - - dtc-process-mappings - - related-mappings - - rule-mappings - - dtc-rule-mappings - - server-mappings - - service-mappings - - dtc-service-mappings - - source-mappings - - dtc-source-mappings - - pb-override-source-mappings - - threat-mappings - - tls-mappings - - url-mappings - - user_agent-mappings - - dtc-user_agent-mappings - - common-settings - - common-dynamic-mappings - data_stream: {} - ignore_missing_component_templates: [] - index_patterns: - - logs-soc-so* - priority: 500 - template: - mappings: - date_detection: false - dynamic_templates: - - strings_as_keyword: - mapping: - ignore_above: 1024 - type: keyword - match_mapping_type: string - settings: - index: - lifecycle: - name: so-logs-soc-logs - mapping: - total_fields: - limit: 5000 - number_of_replicas: 0 - number_of_shards: 1 - refresh_interval: 30s - sort: - field: '@timestamp' - order: desc - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - warm: 7 so-logs-sonicwall_firewall_x_log: index_sorting: false index_template: @@ -8780,8 +8520,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sonicwall_firewall.log@custom index_patterns: - logs-sonicwall_firewall.log-* priority: 501 @@ -8826,8 +8564,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sophos_central.alert@custom index_patterns: - logs-sophos_central.alert-* priority: 501 @@ -8872,8 +8608,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sophos_central.event@custom index_patterns: - logs-sophos_central.event-* priority: 501 @@ -8918,8 +8652,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sophos.utm@custom index_patterns: - logs-sophos.utm-* priority: 501 @@ -8964,8 +8696,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-sophos.xg@custom index_patterns: - logs-sophos.xg-* priority: 501 @@ -9010,8 +8740,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-symantec_endpoint.log@custom index_patterns: - logs-symantec_endpoint.log-* priority: 501 @@ -9057,8 +8785,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-system.application@custom index_patterns: - logs-system.application* priority: 501 @@ -9104,8 +8830,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-system.auth@custom index_patterns: - logs-system.auth* priority: 501 @@ -9151,8 +8875,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-system.security@custom index_patterns: - logs-system.security* priority: 501 @@ -9198,8 +8920,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-system.syslog@custom index_patterns: - logs-system.syslog* priority: 501 @@ -9245,8 +8965,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-system.system@custom index_patterns: - logs-system.system* priority: 501 @@ -9291,8 +9009,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-tenable_sc.asset@custom index_patterns: - logs-tenable_sc.asset-* priority: 501 @@ -9337,8 +9053,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-tenable_sc.plugin@custom index_patterns: - logs-tenable_sc.plugin-* priority: 501 @@ -9383,8 +9097,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-tenable_sc.vulnerability@custom index_patterns: - logs-tenable_sc.vulnerability-* priority: 501 @@ -9429,8 +9141,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_abusech.malware@custom index_patterns: - logs-ti_abusech.malware-* priority: 501 @@ -9475,8 +9185,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_abusech.malwarebazaar@custom index_patterns: - logs-ti_abusech.malwarebazaar-* priority: 501 @@ -9521,8 +9229,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_abusech.threatfox@custom index_patterns: - logs-ti_abusech.threatfox-* priority: 501 @@ -9567,8 +9273,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_abusech.url@custom index_patterns: - logs-ti_abusech.url-* priority: 501 @@ -9603,27 +9307,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_anomali_x_threatstream: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-ti_anomali.threatstream@package - - logs-ti_anomali.threatstream@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-ti_anomali.threatstream@custom index_patterns: - - logs-ti_anomali.threatstream-* - priority: 501 + - "logs-ti_anomali.threatstream-*" template: settings: index: lifecycle: name: so-logs-ti_anomali.threatstream-logs number_of_replicas: 0 + composed_of: + - "logs-ti_anomali.threatstream@package" + - "logs-ti_anomali.threatstream@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -9649,27 +9351,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_cybersixgill_x_threat: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-ti_cybersixgill.threat@package - - logs-ti_cybersixgill.threat@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-ti_cybersixgill.threat@custom index_patterns: - - logs-ti_cybersixgill.threat-* - priority: 501 + - "logs-ti_cybersixgill.threat-*" template: settings: index: lifecycle: name: so-logs-ti_cybersixgill.threat-logs number_of_replicas: 0 + composed_of: + - "logs-ti_cybersixgill.threat@package" + - "logs-ti_cybersixgill.threat@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -9705,8 +9405,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_misp.threat@custom index_patterns: - logs-ti_misp.threat-* priority: 501 @@ -9751,8 +9449,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_misp.threat_attributes@custom index_patterns: - logs-ti_misp.threat_attributes-* priority: 501 @@ -9797,8 +9493,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_otx.pulses_subscribed@custom index_patterns: - logs-ti_otx.pulses_subscribed-* priority: 501 @@ -9843,8 +9537,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_otx.threat@custom index_patterns: - logs-ti_otx.threat-* priority: 501 @@ -9889,8 +9581,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_recordedfuture.latest_ioc-template@custom index_patterns: - logs-ti_recordedfuture.latest_ioc-template-* priority: 501 @@ -9935,8 +9625,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-ti_recordedfuture.threat@custom index_patterns: - logs-ti_recordedfuture.threat-* priority: 501 @@ -9971,27 +9659,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-ti_threatq_x_threat: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-ti_threatq.threat@package - - logs-ti_threatq.threat@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-ti_threatq.threat@custom index_patterns: - - logs-ti_threatq.threat-* - priority: 501 + - "logs-ti_threatq.threat-*" template: settings: index: lifecycle: name: so-logs-ti_threatq.threat-logs number_of_replicas: 0 + composed_of: + - "logs-ti_threatq.threat@package" + - "logs-ti_threatq.threat@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -10017,27 +9703,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-vsphere_x_log: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-vsphere.log@package - - logs-vsphere.log@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-vsphere.log@custom index_patterns: - - logs-vsphere.log-* - priority: 501 + - "logs-vsphere.log-*" template: settings: index: lifecycle: name: so-logs-vsphere.log-logs number_of_replicas: 0 + composed_of: + - "logs-vsphere.log@package" + - "logs-vsphere.log@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -10073,8 +9757,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-windows.forwarded@custom index_patterns: - logs-windows.forwarded* priority: 501 @@ -10119,8 +9801,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-windows.powershell@custom index_patterns: - logs-windows.powershell-* priority: 501 @@ -10165,8 +9845,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-windows.powershell_operational@custom index_patterns: - logs-windows.powershell_operational-* priority: 501 @@ -10211,8 +9889,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-windows.sysmon_operational@custom index_patterns: - logs-windows.sysmon_operational-* priority: 501 @@ -10247,28 +9923,25 @@ elasticsearch: priority: 50 min_age: 30d so-logs-winlog_x_winlog: - index_sorting: false + index_sorting: False index_template: - composed_of: - - logs-winlog.winlog@package - - logs-winlog.winlog@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - logs-winlog.winlog@package - - logs-winlog.winlog@custom index_patterns: - - logs-winlog.winlog-* - priority: 501 + - "logs-winlog.winlog-*" template: settings: index: lifecycle: name: so-logs-winlog.winlog-logs number_of_replicas: 0 + composed_of: + - "logs-winlog.winlog@package" + - "logs-winlog.winlog@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false policy: phases: cold: @@ -10304,8 +9977,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zia.alerts@custom index_patterns: - logs-zscaler_zia.alerts-* priority: 501 @@ -10350,8 +10021,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zia.dns@custom index_patterns: - logs-zscaler_zia.dns-* priority: 501 @@ -10396,8 +10065,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zia.firewall@custom index_patterns: - logs-zscaler_zia.firewall-* priority: 501 @@ -10442,8 +10109,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zia.tunnel@custom index_patterns: - logs-zscaler_zia.tunnel-* priority: 501 @@ -10488,8 +10153,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zia.web@custom index_patterns: - logs-zscaler_zia.web-* priority: 501 @@ -10534,8 +10197,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zpa.app_connector_status@custom index_patterns: - logs-zscaler_zpa.app_connector_status-* priority: 501 @@ -10580,8 +10241,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zpa.audit@custom index_patterns: - logs-zscaler_zpa.audit-* priority: 501 @@ -10626,8 +10285,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zpa.browser_access@custom index_patterns: - logs-zscaler_zpa.browser_access-* priority: 501 @@ -10672,8 +10329,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zpa.user_activity@custom index_patterns: - logs-zscaler_zpa.user_activity-* priority: 501 @@ -10718,8 +10373,6 @@ elasticsearch: data_stream: allow_custom_routing: false hidden: false - ignore_missing_component_templates: - - logs-zscaler_zpa.user_status@custom index_patterns: - logs-zscaler_zpa.user_status-* priority: 501 @@ -10753,6 +10406,317 @@ elasticsearch: set_priority: priority: 50 min_age: 30d + so-metrics-endpoint_x_metadata: + index_sorting: False + index_template: + index_patterns: + - "metrics-endpoint.metadata-*" + template: + settings: + index: + lifecycle: + name: so-metrics-endpoint.metadata-logs + number_of_replicas: 0 + composed_of: + - "metrics-endpoint.metadata@package" + - "metrics-endpoint.metadata@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-endpoint_x_metrics: + index_sorting: False + index_template: + index_patterns: + - "metrics-endpoint.metrics-*" + template: + settings: + index: + lifecycle: + name: so-metrics-endpoint.metrics-logs + number_of_replicas: 0 + composed_of: + - "metrics-endpoint.metrics@package" + - "metrics-endpoint.metrics@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-endpoint_x_policy: + index_sorting: False + index_template: + index_patterns: + - "metrics-endpoint.policy-*" + template: + settings: + index: + lifecycle: + name: so-metrics-endpoint.policy-logs + number_of_replicas: 0 + composed_of: + - "metrics-endpoint.policy@package" + - "metrics-endpoint.policy@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-vsphere_x_datastore: + index_sorting: False + index_template: + index_patterns: + - "metrics-vsphere.datastore-*" + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.datastore-logs + number_of_replicas: 0 + composed_of: + - "metrics-vsphere.datastore@package" + - "metrics-vsphere.datastore@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-vsphere_x_host: + index_sorting: False + index_template: + index_patterns: + - "metrics-vsphere.host-*" + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.host-logs + number_of_replicas: 0 + composed_of: + - "metrics-vsphere.host@package" + - "metrics-vsphere.host@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-metrics-vsphere_x_virtualmachine: + index_sorting: False + index_template: + index_patterns: + - "metrics-vsphere.virtualmachine-*" + template: + settings: + index: + lifecycle: + name: so-metrics-vsphere.virtualmachine-logs + number_of_replicas: 0 + composed_of: + - "metrics-vsphere.virtualmachine@package" + - "metrics-vsphere.virtualmachine@custom" + - "so-fleet_globals-1" + - "so-fleet_agent_id_verification-1" + priority: 501 + data_stream: + hidden: false + allow_custom_routing: false + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d + so-kismet: + index_sorting: false + index_template: + composed_of: + - kismet-mappings + - source-mappings + - client-mappings + - device-mappings + - network-mappings + - so-fleet_globals-1 + - so-fleet_agent_id_verification-1 + data_stream: + allow_custom_routing: false + hidden: false + index_patterns: + - logs-kismet-so* + priority: 501 + template: + settings: + index: + lifecycle: + name: so-kismet-logs + number_of_replicas: 0 + policy: + phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d + delete: + actions: + delete: {} + min_age: 365d + hot: + actions: + rollover: + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 + min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-logstash: index_sorting: false index_template: @@ -10814,7 +10778,6 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings - ignore_missing_component_templates: [] index_patterns: - logs-logstash-default* priority: 500 @@ -10864,374 +10827,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - so-metrics-endpoint_x_metadata: - index_sorting: false - index_template: - composed_of: - - metrics-endpoint.metadata@package - - metrics-endpoint.metadata@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-endpoint.metadata@custom - index_patterns: - - metrics-endpoint.metadata-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-endpoint.metadata-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-endpoint_x_metrics: - index_sorting: false - index_template: - composed_of: - - metrics-endpoint.metrics@package - - metrics-endpoint.metrics@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-endpoint.metrics@custom - index_patterns: - - metrics-endpoint.metrics-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-endpoint.metrics-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-endpoint_x_policy: - index_sorting: false - index_template: - composed_of: - - metrics-endpoint.policy@package - - metrics-endpoint.policy@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-endpoint.policy@custom - index_patterns: - - metrics-endpoint.policy-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-endpoint.policy-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-fleet_server_x_agent_status: - index_sorting: false - index_template: - composed_of: - - metrics@tsdb-settings - - metrics-fleet_server.agent_status@package - - metrics-fleet_server.agent_status@custom - - ecs@mappings - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-fleet_server.agent_status@custom - index_patterns: - - metrics-fleet_server.agent_status-* - priority: 501 - template: - settings: - index: - mode: time_series - number_of_replicas: 0 - so-metrics-fleet_server_x_agent_versions: - index_sorting: false - index_template: - composed_of: - - metrics@tsdb-settings - - metrics-fleet_server.agent_versions@package - - metrics-fleet_server.agent_versions@custom - - ecs@mappings - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-fleet_server.agent_versions@custom - index_patterns: - - metrics-fleet_server.agent_versions-* - priority: 501 - template: - settings: - index: - mode: time_series - number_of_replicas: 0 - so-metrics-nginx_x_stubstatus: - index_sorting: false - index_template: - composed_of: - - metrics-nginx.stubstatus@package - - metrics-nginx.stubstatus@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-nginx.stubstatus@custom - index_patterns: - - metrics-nginx.stubstatus-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-nginx.stubstatus-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_datastore: - index_sorting: false - index_template: - composed_of: - - metrics-vsphere.datastore@package - - metrics-vsphere.datastore@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-vsphere.datastore@custom - index_patterns: - - metrics-vsphere.datastore-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.datastore-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_host: - index_sorting: false - index_template: - composed_of: - - metrics-vsphere.host@package - - metrics-vsphere.host@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-vsphere.host@custom - index_patterns: - - metrics-vsphere.host-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.host-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d - so-metrics-vsphere_x_virtualmachine: - index_sorting: false - index_template: - composed_of: - - metrics-vsphere.virtualmachine@package - - metrics-vsphere.virtualmachine@custom - - so-fleet_globals-1 - - so-fleet_agent_id_verification-1 - data_stream: - allow_custom_routing: false - hidden: false - ignore_missing_component_templates: - - metrics-vsphere.virtualmachine@custom - index_patterns: - - metrics-vsphere.virtualmachine-* - priority: 501 - template: - settings: - index: - lifecycle: - name: so-metrics-vsphere.virtualmachine-logs - number_of_replicas: 0 - policy: - phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d - hot: - actions: - rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 - min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d so-redis: index_sorting: false index_template: @@ -11293,7 +10888,6 @@ elasticsearch: - vulnerability-mappings - common-settings - common-dynamic-mappings - ignore_missing_component_templates: [] index_patterns: - logs-redis-default* priority: 500 @@ -11406,7 +11000,6 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} - ignore_missing_component_templates: [] index_patterns: - logs-strelka-so* priority: 500 @@ -11518,7 +11111,6 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} - ignore_missing_component_templates: [] index_patterns: - logs-suricata-so* priority: 500 @@ -11630,7 +11222,6 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} - ignore_missing_component_templates: [] index_patterns: - logs-suricata.alerts-* priority: 500 @@ -11743,7 +11334,6 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} - ignore_missing_component_templates: [] index_patterns: - logs-syslog-so* priority: 500 @@ -11857,7 +11447,6 @@ elasticsearch: - common-settings - common-dynamic-mappings data_stream: {} - ignore_missing_component_templates: [] index_patterns: - logs-zeek-so* priority: 500 @@ -11907,87 +11496,6 @@ elasticsearch: set_priority: priority: 50 min_age: 30d - pipelines: - custom001: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom001 - - pipeline: - name: common - custom002: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom002 - - pipeline: - name: common - custom003: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom003 - - pipeline: - name: common - custom004: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom004 - - pipeline: - name: common - custom005: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom005 - - pipeline: - name: common - custom006: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom006 - - pipeline: - name: common - custom007: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom007 - - pipeline: - name: common - custom008: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom008 - - pipeline: - name: common - custom009: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom009 - - pipeline: - name: common - custom010: - description: Custom Pipeline - processors: - - set: - field: tags - value: custom010 - - pipeline: - name: common retention: retention_pct: 50 so_roles: diff --git a/salt/elasticsearch/soc_elasticsearch.yaml b/salt/elasticsearch/soc_elasticsearch.yaml index 085aab7f0..f56ed313e 100644 --- a/salt/elasticsearch/soc_elasticsearch.yaml +++ b/salt/elasticsearch/soc_elasticsearch.yaml @@ -530,58 +530,6 @@ elasticsearch: so-strelka: *indexSettings so-syslog: *indexSettings so-zeek: *indexSettings - so-metrics-fleet_server_x_agent_status: &fleetMetricsSettings - index_sorting: - description: Sorts the index by event time, at the cost of additional processing resource consumption. - advanced: True - readonly: True - helpLink: elasticsearch.html - index_template: - ignore_missing_component_templates: - description: Ignore component templates if they aren't in Elasticsearch. - advanced: True - readonly: True - helpLink: elasticsearch.html - index_patterns: - description: Patterns for matching multiple indices or tables. - advanced: True - readonly: True - helpLink: elasticsearch.html - template: - settings: - index: - mode: - description: Type of mode used for this index. Time series indices can be used for metrics to reduce necessary storage. - advanced: True - readonly: True - helpLink: elasticsearch.html - number_of_replicas: - description: Number of replicas required for this index. Multiple replicas protects against data loss, but also increases storage costs. - advanced: True - readonly: True - helpLink: elasticsearch.html - composed_of: - description: The index template is composed of these component templates. - advanced: True - readonly: True - helpLink: elasticsearch.html - priority: - description: The priority of the index template. - advanced: True - readonly: True - helpLink: elasticsearch.html - data_stream: - hidden: - description: Hide the data stream. - advanced: True - readonly: True - helpLink: elasticsearch.html - allow_custom_routing: - description: Allow custom routing for the data stream. - advanced: True - readonly: True - helpLink: elasticsearch.html - so-metrics-fleet_server_x_agent_versions: *fleetMetricsSettings so_roles: so-manager: &soroleSettings config: diff --git a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load index 12ef4dbf6..080348522 100755 --- a/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load +++ b/salt/elasticsearch/tools/sbin_jinja/so-elasticsearch-templates-load @@ -134,7 +134,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then TEMPLATE=${i::-14} COMPONENT_PATTERN=${TEMPLATE:3} MATCH=$(echo "$TEMPLATE" | grep -E "^so-logs-|^so-metrics" | grep -vE "detections|osquery") - if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" && ! "$COMPONENT_PATTERN" =~ logs-http_endpoint\.generic|logs-winlog\.winlog ]]; then + if [[ -n "$MATCH" && ! "$COMPONENT_LIST" =~ "$COMPONENT_PATTERN" ]]; then load_failures=$((load_failures+1)) echo "Component template does not exist for $COMPONENT_PATTERN. The index template will not be loaded. Load failures: $load_failures" else @@ -153,7 +153,7 @@ if [ ! -f $STATE_FILE_SUCCESS ]; then cd - >/dev/null if [[ $load_failures -eq 0 ]]; then - echo "All templates loaded successfully" + echo "All template loaded successfully" touch $STATE_FILE_SUCCESS else echo "Encountered $load_failures templates that were unable to load, likely due to missing dependencies that will be available later; will retry on next highstate" diff --git a/salt/kibana/files/config_saved_objects.ndjson b/salt/kibana/files/config_saved_objects.ndjson index 50e2ba45c..bc503debb 100644 --- a/salt/kibana/files/config_saved_objects.ndjson +++ b/salt/kibana/files/config_saved_objects.ndjson @@ -1 +1 @@ -{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.14.2","id": "8.14.2","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} +{"attributes": {"buildNum": 39457,"defaultIndex": "logs-*","defaultRoute": "/app/dashboards#/view/a8411b30-6d03-11ea-b301-3d6c35840645","discover:sampleSize": 100,"theme:darkMode": true,"timepicker:timeDefaults": "{\n \"from\": \"now-24h\",\n \"to\": \"now\"\n}"},"coreMigrationVersion": "8.10.4","id": "8.10.4","references": [],"type": "config","updated_at": "2021-10-10T10:10:10.105Z","version": "WzI5NzUsMl0="} diff --git a/salt/kibana/tools/sbin_jinja/so-kibana-config-load b/salt/kibana/tools/sbin_jinja/so-kibana-config-load index fc0896009..5b4b52560 100644 --- a/salt/kibana/tools/sbin_jinja/so-kibana-config-load +++ b/salt/kibana/tools/sbin_jinja/so-kibana-config-load @@ -63,7 +63,7 @@ update() { IFS=$'\r\n' GLOBIGNORE='*' command eval 'LINES=($(cat $1))' for i in "${LINES[@]}"; do - RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.14.2" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") + RESPONSE=$(curl -K /opt/so/conf/elasticsearch/curl.config -X PUT "localhost:5601/api/saved_objects/config/8.10.4" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d " $i ") echo $RESPONSE; if [[ "$RESPONSE" != *"\"success\":true"* ]] && [[ "$RESPONSE" != *"updated_at"* ]] ; then RETURN_CODE=1;fi done From a23789287e86033a9e35726f8a1c4b9f6d69102a Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 15 Jul 2024 11:29:47 -0400 Subject: [PATCH 58/80] force var to be list of string --- salt/suricata/soc_suricata.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/salt/suricata/soc_suricata.yaml b/salt/suricata/soc_suricata.yaml index 1ecabacd8..f865468e1 100644 --- a/salt/suricata/soc_suricata.yaml +++ b/salt/suricata/soc_suricata.yaml @@ -154,12 +154,14 @@ suricata: description: Assign a list of hosts, or networks, using CIDR notation, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable. regex: ^(((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\/([0-9]|[1-2][0-9]|3[0-2]))?$|^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?))|:))|(([0-9A-Fa-f]{1,4}:){5}((:[0-9A-Fa-f]{1,4}){1,2}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){4}((:[0-9A-Fa-f]{1,4}){1,3}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){3}((:[0-9A-Fa-f]{1,4}){1,4}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){2}((:[0-9A-Fa-f]{1,4}){1,5}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(([0-9A-Fa-f]{1,4}:){1}((:[0-9A-Fa-f]{1,4}){1,6}|:((25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)\.){3}(25[0-5]|(2[0-4]|1[0-9])[0-9]|0?[0-9][0-9]?)|:))|(:((:[0-9A-Fa-f]{1,4}){1,7}|:)))(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$ regexFailureMessage: You must enter a valid IP address or CIDR. - helpLink: suricata.html + forcedType: "[]string" duplicates: True + helpLink: suricata.html EXTERNAL_NET: &suriaddressgroup description: Assign a list of hosts, or networks, or other customization, to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable. - helpLink: suricata.html + forcedType: "[]string" duplicates: True + helpLink: suricata.html HTTP_SERVERS: *suriaddressgroup SMTP_SERVERS: *suriaddressgroup SQL_SERVERS: *suriaddressgroup @@ -176,8 +178,9 @@ suricata: port-groups: HTTP_PORTS: &suriportgroup description: Assign a list of network port numbers to this Suricata variable. The variable can then be re-used within Suricata rules. This allows for a single adjustment to the variable that will then affect all rules referencing the variable. - helpLink: suricata.html + forcedType: "[]string" duplicates: True + helpLink: suricata.html SHELLCODE_PORTS: *suriportgroup ORACLE_PORTS: *suriportgroup SSH_PORTS: *suriportgroup From 4e81860a13486d272b9395301822f0b2b69da4c4 Mon Sep 17 00:00:00 2001 From: weslambert Date: Mon, 15 Jul 2024 11:33:52 -0400 Subject: [PATCH 59/80] Revert "Change pipeline version for agent" --- .../component/elastic-agent/logs-elastic_agent@package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json index efd85bb4b..7bcc34de9 100644 --- a/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json +++ b/salt/elasticsearch/templates/component/elastic-agent/logs-elastic_agent@package.json @@ -6,7 +6,7 @@ "name": "logs" }, "codec": "best_compression", - "default_pipeline": "logs-elastic_agent-1.20.0", + "default_pipeline": "logs-elastic_agent-1.13.1", "mapping": { "total_fields": { "limit": "10000" From 3991c7b5fea285f2277a4a03a4165b720eb294ac Mon Sep 17 00:00:00 2001 From: Doug Burks Date: Mon, 15 Jul 2024 15:52:00 -0400 Subject: [PATCH 60/80] FEATURE: Add new action to SOC Actions list to allow users to more easily add their own actions #13346 --- salt/soc/defaults.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index ce183b862..6ddf534c7 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -96,6 +96,12 @@ soc: links: - '/#/alerts?q=rule.uuid: {:so_detection.publicId|escape} | groupby rule.name event.module* event.severity_label' target: '' + - name: actionAdd + description: actionAddHelp + icon: fa-plus + links: + - '/#/config?s=soc.config.actions' + target: '' eventFields: default: - soc_timestamp From 542a116b8c1c77bd3943acaf7b00d5fc3291a756 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:26:52 -0400 Subject: [PATCH 61/80] use so-yaml add for kafka pillar change Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/manager/tools/sbin/soup | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 0ace4a5b2..371b447bb 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -677,11 +677,10 @@ up_to_2.4.90() { # rearranging the kafka pillar to reduce clutter in SOC UI kafkasavedpass=$(so-yaml.py get /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password) kafkatrimpass=$(echo $kafkasavedpass | awk '{print $1}') + echo "Making changes to the Kafka pillar layout" so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password - echo ' config:' >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls - echo ' password: '$kafkatrimpass >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls - echo ' trustpass: '$kafkatrust >> /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls - + so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password $kafkatrimpass + so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass $kafkatrust INSTALLEDVERSION=2.4.90 } From 1b623c5c7ae5b0e22b5de97eb4ba6e7dfa6c12e4 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:27:48 -0400 Subject: [PATCH 62/80] Show Kafka EPS for nodes with broker role only Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- .../templates/dashboard-security_onion_performance.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/influxdb/templates/dashboard-security_onion_performance.json b/salt/influxdb/templates/dashboard-security_onion_performance.json index 4f543c8d1..835aedb03 100644 --- a/salt/influxdb/templates/dashboard-security_onion_performance.json +++ b/salt/influxdb/templates/dashboard-security_onion_performance.json @@ -1 +1 @@ -[{"apiVersion":"influxdata.com/v2alpha1","kind":"Dashboard","metadata":{"name":"vivid-wilson-002001"},"spec":{"charts":[{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Uptime","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime\")\n |> hostFilter()\n |> map(fn: (r) => ({r with _value: r._value / (24 * 60 * 60)}))\n |> group(columns: [\"host\"])\n |> last()\n |> lowestMin(n:1)"}],"staticLegend":{},"suffix":" days","width":1},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"},{"id":"z83MTSufTrlrCoEPiBXda","name":"ruby","type":"text","hex":"#BF3D5E","value":1}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Critical Alarms","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> filter(fn: (r) => r[\"_level\"] == \"crit\")\n |> count()"}],"staticLegend":{},"suffix":" ","width":1,"yPos":2},{"colors":[{"id":"base","name":"rainforest","type":"text","hex":"#4ED8A0"},{"id":"QCTYWuGuHkikYFsZSKMzQ","name":"rainforest","type":"text","hex":"#4ED8A0"},{"id":"QdpMyTRBb0LJ56-P5wfAW","name":"laser","type":"text","hex":"#00C9FF","value":1},{"id":"VQGwCoMrxZyP8asiOW5Cq","name":"tiger","type":"text","hex":"#F48D38","value":2},{"id":"zSO9QkesSIxrU_ntCBx2i","name":"ruby","type":"text","hex":"#BF3D5E","value":3}],"fieldOptions":[{"fieldName":"_time","visible":true},{"displayName":"Alarm","fieldName":"_check_name","visible":true},{"displayName":"Severity","fieldName":"_value","visible":true},{"displayName":"Status","fieldName":"_level","visible":true}],"height":6,"kind":"Table","name":"Alarm Status","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> drop(columns: [\"_value\"])\n |> duplicate(column: \"_level\", as: \"_value\")\n |> map(fn: (r) => ({ r with _value: if r._value == \"ok\" then 0 else if r._value == \"info\" then 1 else if r._value == \"warn\" then 2 else 3 }))\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> keep(columns: [\"_check_name\",\"_level\",\"_value\"])"}],"staticLegend":{},"tableOptions":{"sortBy":"_check_name","verticalTimeAxis":true},"timeFormat":"YYYY-MM-DD HH:mm:ss","width":3,"yPos":4},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"B"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Elasticsearch Storage Size","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"store_size_in_bytes\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"mean\")"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"store_size_in_bytes\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":10},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"B"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"InfluxDB Size","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"influxsize\")\n |> filter(fn: (r) => r[\"_field\"] == \"kbytes\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 1000.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"mean\")"},{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"influxsize\")\n |> filter(fn: (r) => r[\"_field\"] == \"kbytes\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 1000.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":14},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":" days"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"System Uptime","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24 * 60 * 60)}))\n |> yield(name: \"last\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24 * 60 * 60)}))\n |> yield(name: \"Trend\")"}],"shade":true,"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":18},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"lQ75rvTyd2Lq5pZjzy6LB","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"KLfpRZtiEnU2GxjPtrrzQ","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"1kLynwKxvJ3B5IeJnrBqp","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Kafka EPS","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_topics\")\n |> filter(fn: (r) => r[\"_field\"] == \"MessagesInPerSec.Count\")\n |> derivative(unit: 1s, nonNegative: true)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_topics\")\n |> filter(fn: (r) => r[\"_field\"] == \"MessagesInPerSec.Count\")\n |> derivative(unit: 1s, nonNegative: true)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":22},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"System CPU Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_idle\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: r._value * -1.0 + 100.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_idle\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\",\"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: r._value * -1.0 + 100.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":26},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"QDwChKZWuQV0BaJcEeSam","name":"Atlantis","type":"scale","hex":"#74D495"},{"id":"ThD0WTqKHltQEVlq9mo6K","name":"Atlantis","type":"scale","hex":"#3F3FBA"},{"id":"FBHYZiwDLKyQK3eRfUD-0","name":"Atlantis","type":"scale","hex":"#FF4D9E"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"System Memory Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":30},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Monitor Interface Traffic - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"bytes_recv\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":34},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Management Interface Traffic - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"bytes_recv\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":6,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":38},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Stenographer Packet Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"stenodrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"stenodrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":42},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Disk Usage /","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":46},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"5m Load Average","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"load5\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"width":1,"xPos":1},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"},{"id":"z83MTSufTrlrCoEPiBXda","name":"tiger","type":"text","hex":"#F48D38","value":1}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Warning Alarms","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> filter(fn: (r) => r[\"_level\"] == \"warn\")\n |> count()"}],"staticLegend":{},"suffix":" ","width":1,"xPos":1,"yPos":2},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"IO Wait","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_iowait\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":2},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"},{"id":"z83MTSufTrlrCoEPiBXda","name":"laser","type":"text","hex":"#00C9FF","value":1}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Informative Alarms","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> filter(fn: (r) => r[\"_level\"] == \"info\")\n |> count()"}],"staticLegend":{},"suffix":" ","width":1,"xPos":2,"yPos":2},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Estimated EPS In","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"in\")\n |> hostFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"width":1,"xPos":3},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"pineapple","type":"threshold","hex":"#FFB94A","value":70},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"fire","type":"threshold","hex":"#DC4E58","value":80},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"CPU Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_idle\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> map(fn: (r) => ({r with _value: r._value * -1.0 + 100.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":3,"yPos":2},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"kOQLOg2H4FVEE-E1_L8Kq","name":"laser","type":"threshold","hex":"#00C9FF","value":85},{"id":"5IArg2lDb8KvnphywgUXa","name":"tiger","type":"threshold","hex":"#F48D38","value":90},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"ruby","type":"threshold","hex":"#BF3D5E","value":95},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"Root Disk Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"path\"] == \"/\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":3,"yPos":6},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Suricata Packet Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"suridrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"suridrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","xPos":3,"yCol":"_value","yPos":42},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Redis Queue","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"redisqueue\")\n |> filter(fn: (r) => r[\"_field\"] == \"unparsed\")\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"width":1,"xPos":4},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Elasticsearch Document Count","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"docs_count\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"mean\")"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"docs_count\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":10},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"heightRatio":0.301556420233463,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Redis Queue","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"redisqueue\")\n |> filter(fn: (r) => r[\"_field\"] == \"unparsed\")\n |> group(columns: [\"host\", \"_field\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"redisqueue\")\n |> filter(fn: (r) => r[\"_field\"] == \"unparsed\")\n |> group(columns: [\"host\", \"_field\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.301556420233463,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":14},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":" days"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container Uptime","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_status\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime_ns\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> group(columns: [\"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24 * 60 * 60 * 1000000000)}))\n |> yield(name: \"last\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_status\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime_ns\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> group(columns: [\"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24.0 * 60.0 * 60.0 * 1000000000.0)}))\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":18},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":2,"hoverDimension":"auto","kind":"Single_Stat_Plus_Line","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Kafka Active Controllers","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_controller\")\n |> filter(fn: (r) => r[\"_field\"] == \"ActiveControllerCount.Value\")\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"current\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_controller\")\n |> filter(fn: (r) => r[\"_field\"] == \"ActiveControllerCount.Value\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":22},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":2,"hoverDimension":"auto","kind":"Single_Stat_Plus_Line","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Kafka Active Brokers","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_controller\")\n |> filter(fn: (r) => r[\"_field\"] == \"ActiveBrokerCount.Value\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"trend\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_controller\")\n |> filter(fn: (r) => r[\"_field\"] == \"ActiveBrokerCount.Value\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"current\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":24},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"yT5vTIlaaFChSrQvKLfqf","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"mzzUVSu3ibTph1JmQmDAQ","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"mOcnDo7l8ii6qNLFIB5rs","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container CPU Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"mean\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":26},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"QDwChKZWuQV0BaJcEeSam","name":"Atlantis","type":"scale","hex":"#74D495"},{"id":"ThD0WTqKHltQEVlq9mo6K","name":"Atlantis","type":"scale","hex":"#3F3FBA"},{"id":"FBHYZiwDLKyQK3eRfUD-0","name":"Atlantis","type":"scale","hex":"#FF4D9E"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container Memory Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"mean\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":30},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b"}],"colorizeRows":true,"colors":[{"id":"0ynR6Zs0wuQ3WY0Lz-_KC","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"YiArehCNBwFm9mn8DSXSG","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"DxByY_EQW9Xs2jD5ktkG5","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container Traffic - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_net\")\n |> filter(fn: (r) => r[\"_field\"] == \"rx_bytes\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"mean\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_net\")\n |> filter(fn: (r) => r[\"_field\"] == \"rx_bytes\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with _value: r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":34},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Disk Usage /nsm","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/nsm\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/nsm\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xPos":4,"yPos":46},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Inbound Traffic","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\") \n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: r._value * 8.0 / (1000.0 * 1000.0)}))\n |> group(columns: [\"host\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> last()\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":" Mb/s","width":1,"xPos":5},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Inbound Drops","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop_in\") \n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: r._value * 8.0 / (1000.0 * 1000.0)}))\n |> group(columns: [\"host\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> last()\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":" Mb/s","width":1,"xPos":6},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"pineapple","type":"threshold","hex":"#FFB94A","value":70},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"fire","type":"threshold","hex":"#DC4E58","value":80},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"Memory Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":6,"yPos":2},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"laser","type":"threshold","hex":"#00C9FF","value":85},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"tiger","type":"threshold","hex":"#F48D38","value":90},{"id":"H7uprvKmMEh39en6X-ms_","name":"ruby","type":"threshold","hex":"#BF3D5E","value":95},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"NSM Disk Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"path\"] == \"/nsm\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":6,"yPos":6},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Management Interface Traffic - Outbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_sent\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n \n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"bytes_sent\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_sent\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n \n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":6,"widthRatio":1,"xCol":"_time","xPos":6,"yCol":"_value","yPos":38},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Zeek Packet Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekdrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekdrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","xPos":6,"yCol":"_value","yPos":42},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Capture Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekcaptureloss\")\n |> filter(fn: (r) => r[\"_field\"] == \"loss\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":7},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Zeek Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekdrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":8},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"s"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"heightRatio":0.301556420233463,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Elastic Ingest Time Spent","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_community_id_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"community.id_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_conditional_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"conditional_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_date_index_name_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"date.index.name_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_date_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"date_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_dissect_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"dissect_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_dot_expander_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"dot.expander_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_geoip_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"geoip_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_grok_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"grok_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_json_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"json_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_kv_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"kv_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_lowercase_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"lowercase_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_rename_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"rename_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_script_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"script_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_user_agent_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"user.agent_time\")"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.301556420233463,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":10},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"1m Load Average","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"load1\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"load1\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\",\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":14,"yTickStep":1},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":" e/s"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"heightRatio":0.301556420233463,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Logstash EPS","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"in\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"out\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: -r._value}))\n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"in\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"out\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: -r._value}))\n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.301556420233463,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":18},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":4,"hoverDimension":"auto","kind":"Single_Stat_Plus_Line","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Kafka Under Replicated Partitions","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_partition\")\n |> filter(fn: (r) => r[\"_field\"] == \"UnderReplicatedPartitions\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"partition\",\"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_partition\")\n |> filter(fn: (r) => r[\"_field\"] == \"UnderReplicatedPartitions\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"partition\",\"host\", \"role\"])\n |> yield(name: \"trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":22},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"UAehjIsi65P8u92M_3sQY","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"_SCP8Npp4NVMx2N4mfuzX","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"BoMPg4R1KDp_UsRORdV3_","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"IO Wait","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_iowait\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_iowait\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":26},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"QDwChKZWuQV0BaJcEeSam","name":"Atlantis","type":"scale","hex":"#74D495"},{"id":"ThD0WTqKHltQEVlq9mo6K","name":"Atlantis","type":"scale","hex":"#3F3FBA"},{"id":"FBHYZiwDLKyQK3eRfUD-0","name":"Atlantis","type":"scale","hex":"#FF4D9E"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Swap Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"swap\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"swap\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":30},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Monitor Interface Drops - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop_in\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"drop_in\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop_in\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":34},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":" days"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Stenographer PCAP Retention","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"pcapage\")\n |> filter(fn: (r) => r[\"_field\"] == \"seconds\")\n |> map(fn: (r) => ({ r with _value: r._value / (24.0 * 3600.0)}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])"},{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"pcapage\")\n |> filter(fn: (r) => r[\"_field\"] == \"seconds\")\n |> map(fn: (r) => ({ r with _value: r._value / (24.0 * 3600.0)}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":46},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Suricata Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"suridrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":9},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"pineapple","type":"threshold","hex":"#FFB94A","value":50},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"fire","type":"threshold","hex":"#DC4E58","value":70},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"Swap Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"swap\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":9,"yPos":2},{"colors":[{"id":"base","name":"white","type":"text","hex":"#ffffff"}],"fieldOptions":[{"displayName":"Host","fieldName":"host","visible":true},{"displayName":"Name","fieldName":"container_name","visible":true},{"displayName":"Status","fieldName":"container_status","visible":true},{"displayName":"OOM Killed","fieldName":"_value","visible":true},{"displayName":"_start","fieldName":"_start","visible":true},{"displayName":"_stop","fieldName":"_stop","visible":true},{"displayName":"_time","fieldName":"_time","visible":true},{"displayName":"_field","fieldName":"_field","visible":true},{"displayName":"_measurement","fieldName":"_measurement","visible":true},{"displayName":"engine_host","fieldName":"engine_host","visible":true},{"displayName":"role","fieldName":"role","visible":true},{"displayName":"server_version","fieldName":"server_version","visible":true},{"displayName":"container_image","fieldName":"container_image","visible":true},{"displayName":"container_version","fieldName":"container_version","visible":true},{"displayName":"description","fieldName":"description","visible":true},{"displayName":"maintainer","fieldName":"maintainer","visible":true},{"displayName":"io.k8s.description","fieldName":"io.k8s.description","visible":true},{"displayName":"io.k8s.display-name","fieldName":"io.k8s.display-name","visible":true},{"displayName":"license","fieldName":"license","visible":true},{"displayName":"name","fieldName":"name","visible":true},{"displayName":"org.label-schema.build-date","fieldName":"org.label-schema.build-date","visible":true},{"displayName":"org.label-schema.license","fieldName":"org.label-schema.license","visible":true},{"displayName":"org.label-schema.name","fieldName":"org.label-schema.name","visible":true},{"displayName":"org.label-schema.schema-version","fieldName":"org.label-schema.schema-version","visible":true},{"displayName":"org.label-schema.url","fieldName":"org.label-schema.url","visible":true},{"displayName":"org.label-schema.vcs-ref","fieldName":"org.label-schema.vcs-ref","visible":true},{"displayName":"org.label-schema.vcs-url","fieldName":"org.label-schema.vcs-url","visible":true},{"displayName":"org.label-schema.vendor","fieldName":"org.label-schema.vendor","visible":true},{"displayName":"org.label-schema.version","fieldName":"org.label-schema.version","visible":true},{"displayName":"org.opencontainers.image.created","fieldName":"org.opencontainers.image.created","visible":true},{"displayName":"org.opencontainers.image.licenses","fieldName":"org.opencontainers.image.licenses","visible":true},{"displayName":"org.opencontainers.image.title","fieldName":"org.opencontainers.image.title","visible":true},{"displayName":"org.opencontainers.image.vendor","fieldName":"org.opencontainers.image.vendor","visible":true},{"displayName":"release","fieldName":"release","visible":true},{"displayName":"summary","fieldName":"summary","visible":true},{"displayName":"url","fieldName":"url","visible":true},{"displayName":"vendor","fieldName":"vendor","visible":true},{"displayName":"version","fieldName":"version","visible":true},{"displayName":"org.label-schema.usage","fieldName":"org.label-schema.usage","visible":true},{"displayName":"org.opencontainers.image.documentation","fieldName":"org.opencontainers.image.documentation","visible":true},{"displayName":"org.opencontainers.image.revision","fieldName":"org.opencontainers.image.revision","visible":true},{"displayName":"org.opencontainers.image.source","fieldName":"org.opencontainers.image.source","visible":true},{"displayName":"org.opencontainers.image.url","fieldName":"org.opencontainers.image.url","visible":true},{"displayName":"org.opencontainers.image.version","fieldName":"org.opencontainers.image.version","visible":true},{"displayName":"org.opencontainers.image.description","fieldName":"org.opencontainers.image.description","visible":true}],"height":4,"kind":"Table","name":"Most Recent Container Events","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_status\")\n |> filter(fn: (r) => r[\"_field\"] == \"oomkilled\")\n |> filter(fn: (r) => r[\"container_status\"] != \"running\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"container_name\", \"host\"])\n |> last()\n |> group()\n |> keep(columns: [\"_value\", \"container_name\", \"host\", \"container_status\"])"}],"staticLegend":{},"tableOptions":{"sortBy":"container_name","verticalTimeAxis":true},"timeFormat":"YYYY-MM-DD HH:mm:ss","width":3,"xPos":9,"yPos":6},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Zeek Capture Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekcaptureloss\")\n |> filter(fn: (r) => r[\"_field\"] == \"loss\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekcaptureloss\")\n |> filter(fn: (r) => r[\"_field\"] == \"loss\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","xPos":9,"yCol":"_value","yPos":42},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Stenographer Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"stenodrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":10},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"PCAP Retention","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"pcapage\")\n |> filter(fn: (r) => r[\"_field\"] == \"seconds\")\n |> hostFilter()\n |> map(fn: (r) => ({r with _value: r._value / (24.0 * 60.0 * 60.0)}))\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":" days","width":1,"xPos":11}],"description":"Visualize the Security Onion grid performance metrics and alarm statuses.","name":"Security Onion Performance"}}] \ No newline at end of file +[{"apiVersion":"influxdata.com/v2alpha1","kind":"Dashboard","metadata":{"name":"vivid-wilson-002001"},"spec":{"charts":[{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Uptime","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime\")\n |> hostFilter()\n |> map(fn: (r) => ({r with _value: r._value / (24 * 60 * 60)}))\n |> group(columns: [\"host\"])\n |> last()\n |> lowestMin(n:1)"}],"staticLegend":{},"suffix":" days","width":1},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"},{"id":"z83MTSufTrlrCoEPiBXda","name":"ruby","type":"text","hex":"#BF3D5E","value":1}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Critical Alarms","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> filter(fn: (r) => r[\"_level\"] == \"crit\")\n |> count()"}],"staticLegend":{},"suffix":" ","width":1,"yPos":2},{"colors":[{"id":"base","name":"rainforest","type":"text","hex":"#4ED8A0"},{"id":"QCTYWuGuHkikYFsZSKMzQ","name":"rainforest","type":"text","hex":"#4ED8A0"},{"id":"QdpMyTRBb0LJ56-P5wfAW","name":"laser","type":"text","hex":"#00C9FF","value":1},{"id":"VQGwCoMrxZyP8asiOW5Cq","name":"tiger","type":"text","hex":"#F48D38","value":2},{"id":"zSO9QkesSIxrU_ntCBx2i","name":"ruby","type":"text","hex":"#BF3D5E","value":3}],"fieldOptions":[{"fieldName":"_time","visible":true},{"displayName":"Alarm","fieldName":"_check_name","visible":true},{"displayName":"Severity","fieldName":"_value","visible":true},{"displayName":"Status","fieldName":"_level","visible":true}],"height":6,"kind":"Table","name":"Alarm Status","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> drop(columns: [\"_value\"])\n |> duplicate(column: \"_level\", as: \"_value\")\n |> map(fn: (r) => ({ r with _value: if r._value == \"ok\" then 0 else if r._value == \"info\" then 1 else if r._value == \"warn\" then 2 else 3 }))\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> keep(columns: [\"_check_name\",\"_level\",\"_value\"])"}],"staticLegend":{},"tableOptions":{"sortBy":"_check_name","verticalTimeAxis":true},"timeFormat":"YYYY-MM-DD HH:mm:ss","width":3,"yPos":4},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"B"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Elasticsearch Storage Size","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"store_size_in_bytes\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"mean\")"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"store_size_in_bytes\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":10},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"B"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"InfluxDB Size","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"influxsize\")\n |> filter(fn: (r) => r[\"_field\"] == \"kbytes\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 1000.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"mean\")"},{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"influxsize\")\n |> filter(fn: (r) => r[\"_field\"] == \"kbytes\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 1000.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":14},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":" days"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"System Uptime","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24 * 60 * 60)}))\n |> yield(name: \"last\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24 * 60 * 60)}))\n |> yield(name: \"Trend\")"}],"shade":true,"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":18},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"lQ75rvTyd2Lq5pZjzy6LB","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"KLfpRZtiEnU2GxjPtrrzQ","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"1kLynwKxvJ3B5IeJnrBqp","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Kafka EPS","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"controllerHosts = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_controller\")\n |> filter(fn: (r) => r[\"_field\"] == \"ActiveControllerCount.Value\")\n |> filter(fn: (r) => r[\"_value\"] == 1)\n |> keep(columns: [\"host\"])\n |> distinct(column: \"host\")\n |> map(fn: (r) => ({r with _value: r.host}))\n |> keep(columns: [\"_value\"])\n\ncontrollerHostNames = controllerHosts |> findColumn(fn: (key) => true, column: \"_value\")\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_topics\")\n |> filter(fn: (r) => r[\"_field\"] == \"MessagesInPerSec.Count\")\n |> filter(fn: (r) => not contains(value: r.host, set: controllerHostNames))\n |> derivative(unit: 1s, nonNegative: true)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"controllerHosts = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_controller\")\n |> filter(fn: (r) => r[\"_field\"] == \"ActiveControllerCount.Value\")\n |> filter(fn: (r) => r[\"_value\"] == 1)\n |> keep(columns: [\"host\"])\n |> distinct(column: \"host\")\n |> map(fn: (r) => ({r with _value: r.host}))\n |> keep(columns: [\"_value\"])\n\ncontrollerHostNames = controllerHosts |> findColumn(fn: (key) => true, column: \"_value\")\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_topics\")\n |> filter(fn: (r) => r[\"_field\"] == \"MessagesInPerSec.Count\")\n |> filter(fn: (r) => not contains(value: r.host, set: controllerHostNames))\n |> derivative(unit: 1s, nonNegative: true)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":22},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"System CPU Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_idle\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: r._value * -1.0 + 100.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_idle\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\",\"host\", \"role\"])\n |> map(fn: (r) => ({r with _value: r._value * -1.0 + 100.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":26},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"QDwChKZWuQV0BaJcEeSam","name":"Atlantis","type":"scale","hex":"#74D495"},{"id":"ThD0WTqKHltQEVlq9mo6K","name":"Atlantis","type":"scale","hex":"#3F3FBA"},{"id":"FBHYZiwDLKyQK3eRfUD-0","name":"Atlantis","type":"scale","hex":"#FF4D9E"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"System Memory Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":30},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Monitor Interface Traffic - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"bytes_recv\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":34},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Management Interface Traffic - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"bytes_recv\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":6,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":38},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Stenographer Packet Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"stenodrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"stenodrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":42},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Disk Usage /","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","yCol":"_value","yPos":46},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"5m Load Average","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"load5\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"width":1,"xPos":1},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"},{"id":"z83MTSufTrlrCoEPiBXda","name":"tiger","type":"text","hex":"#F48D38","value":1}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Warning Alarms","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> filter(fn: (r) => r[\"_level\"] == \"warn\")\n |> count()"}],"staticLegend":{},"suffix":" ","width":1,"xPos":1,"yPos":2},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"IO Wait","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_iowait\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":2},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"},{"id":"z83MTSufTrlrCoEPiBXda","name":"laser","type":"text","hex":"#00C9FF","value":1}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Informative Alarms","queries":[{"query":"from(bucket: \"_monitoring\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"statuses\")\n |> filter(fn: (r) => r[\"_field\"] == \"_message\")\n |> group(columns: [\"_check_id\"])\n |> sort(columns: [\"_time\"])\n |> last()\n |> group()\n |> filter(fn: (r) => r[\"_level\"] == \"info\")\n |> count()"}],"staticLegend":{},"suffix":" ","width":1,"xPos":2,"yPos":2},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Estimated EPS In","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"in\")\n |> hostFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"width":1,"xPos":3},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"pineapple","type":"threshold","hex":"#FFB94A","value":70},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"fire","type":"threshold","hex":"#DC4E58","value":80},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"CPU Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_idle\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> map(fn: (r) => ({r with _value: r._value * -1.0 + 100.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":3,"yPos":2},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"kOQLOg2H4FVEE-E1_L8Kq","name":"laser","type":"threshold","hex":"#00C9FF","value":85},{"id":"5IArg2lDb8KvnphywgUXa","name":"tiger","type":"threshold","hex":"#F48D38","value":90},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"ruby","type":"threshold","hex":"#BF3D5E","value":95},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"Root Disk Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"path\"] == \"/\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":3,"yPos":6},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Suricata Packet Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"suridrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"suridrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","xPos":3,"yCol":"_value","yPos":42},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":2,"kind":"Single_Stat","name":"Redis Queue","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"redisqueue\")\n |> filter(fn: (r) => r[\"_field\"] == \"unparsed\")\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"width":1,"xPos":4},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Elasticsearch Document Count","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"docs_count\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"mean\")"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_indices\")\n |> filter(fn: (r) => r[\"_field\"] == \"docs_count\")\n |> filter(fn: (r) => r[\"host\"] == r[\"node_name\"])\n |> hostFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":10},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"heightRatio":0.301556420233463,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Redis Queue","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"redisqueue\")\n |> filter(fn: (r) => r[\"_field\"] == \"unparsed\")\n |> group(columns: [\"host\", \"_field\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"redisqueue\")\n |> filter(fn: (r) => r[\"_field\"] == \"unparsed\")\n |> group(columns: [\"host\", \"_field\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.301556420233463,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":14},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":" days"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container Uptime","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_status\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime_ns\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> group(columns: [\"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24 * 60 * 60 * 1000000000)}))\n |> yield(name: \"last\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_status\")\n |> filter(fn: (r) => r[\"_field\"] == \"uptime_ns\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> group(columns: [\"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> map(fn: (r) => ({r with _value: float(v: r._value) / float(v: 24.0 * 60.0 * 60.0 * 1000000000.0)}))\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":18},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":2,"hoverDimension":"auto","kind":"Single_Stat_Plus_Line","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Kafka Active Controllers","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_controller\")\n |> filter(fn: (r) => r[\"_field\"] == \"ActiveControllerCount.Value\")\n |> aggregateWindow(every: v.windowPeriod, fn: last, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"current\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_controller\")\n |> filter(fn: (r) => r[\"_field\"] == \"ActiveControllerCount.Value\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":22},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":2,"hoverDimension":"auto","kind":"Single_Stat_Plus_Line","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Kafka Active Brokers","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_controller\")\n |> filter(fn: (r) => r[\"_field\"] == \"ActiveBrokerCount.Value\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"trend\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_controller\")\n |> filter(fn: (r) => r[\"_field\"] == \"ActiveBrokerCount.Value\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"current\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":24},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"yT5vTIlaaFChSrQvKLfqf","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"mzzUVSu3ibTph1JmQmDAQ","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"mOcnDo7l8ii6qNLFIB5rs","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container CPU Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"mean\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_cpu\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":26},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"QDwChKZWuQV0BaJcEeSam","name":"Atlantis","type":"scale","hex":"#74D495"},{"id":"ThD0WTqKHltQEVlq9mo6K","name":"Atlantis","type":"scale","hex":"#3F3FBA"},{"id":"FBHYZiwDLKyQK3eRfUD-0","name":"Atlantis","type":"scale","hex":"#FF4D9E"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container Memory Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"mean\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_percent\")\n |> filter(fn: (r) => r[\"container_status\"] == \"running\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":30},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b"}],"colorizeRows":true,"colors":[{"id":"0ynR6Zs0wuQ3WY0Lz-_KC","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"YiArehCNBwFm9mn8DSXSG","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"DxByY_EQW9Xs2jD5ktkG5","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Container Traffic - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_net\")\n |> filter(fn: (r) => r[\"_field\"] == \"rx_bytes\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"mean\")"},{"query":"containerFilter = (tables=<-) =>\n if v.Container != \"(All)\" then\n tables |> filter(fn: (r) => r[\"container_name\"] == v.Container)\n else\n tables\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_net\")\n |> filter(fn: (r) => r[\"_field\"] == \"rx_bytes\")\n |> hostFilter()\n |> roleFilter()\n |> containerFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with _value: r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\", \"container_name\"])\n |> sort(columns: [\"_time\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":4,"yCol":"_value","yPos":34},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Disk Usage /nsm","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/nsm\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> filter(fn: (r) => r[\"path\"] == \"/nsm\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xPos":4,"yPos":46},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Inbound Traffic","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_recv\") \n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: r._value * 8.0 / (1000.0 * 1000.0)}))\n |> group(columns: [\"host\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> last()\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":" Mb/s","width":1,"xPos":5},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Inbound Drops","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop_in\") \n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: r._value * 8.0 / (1000.0 * 1000.0)}))\n |> group(columns: [\"host\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> last()\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":" Mb/s","width":1,"xPos":6},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"pineapple","type":"threshold","hex":"#FFB94A","value":70},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"fire","type":"threshold","hex":"#DC4E58","value":80},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"Memory Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"mem\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":6,"yPos":2},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"laser","type":"threshold","hex":"#00C9FF","value":85},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"tiger","type":"threshold","hex":"#F48D38","value":90},{"id":"H7uprvKmMEh39en6X-ms_","name":"ruby","type":"threshold","hex":"#BF3D5E","value":95},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"NSM Disk Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"disk\")\n |> filter(fn: (r) => r[\"path\"] == \"/nsm\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":6,"yPos":6},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Management Interface Traffic - Outbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_sent\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n \n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"bytes_sent\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"manint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"bytes_sent\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n \n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":6,"widthRatio":1,"xCol":"_time","xPos":6,"yCol":"_value","yPos":38},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Zeek Packet Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekdrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekdrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> hostFilter()\n |> roleFilter()\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","xPos":6,"yCol":"_value","yPos":42},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Capture Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekcaptureloss\")\n |> filter(fn: (r) => r[\"_field\"] == \"loss\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":7},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Zeek Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekdrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":8},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"s"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"heightRatio":0.301556420233463,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Elastic Ingest Time Spent","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_community_id_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"community.id_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_conditional_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"conditional_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_date_index_name_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"date.index.name_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_date_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"date_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_dissect_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"dissect_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_dot_expander_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"dot.expander_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_geoip_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"geoip_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_grok_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"grok_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_json_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"json_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_kv_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"kv_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_lowercase_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"lowercase_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_rename_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"rename_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_script_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"script_time\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"elasticsearch_clusterstats_nodes\")\n |> filter(fn: (r) => r.role == \"standalone\" or r.role == \"eval\" or r.role == \"import\" or r.role == \"managersearch\" or r.role == \"search\" or r.role == \"node\" or r.role == \"heavynode\")\n |> filter(fn: (r) => r[\"_field\"] == \"ingest_processor_stats_user_agent_time_in_millis\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"user.agent_time\")"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.301556420233463,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":10},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"sW2GqpGAsGB5Adx16jKjp","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"TsdXuXwdI5Npi9S8L4f-i","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"OGL29-SUbJ6FyQb0JzbaD","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"1m Load Average","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"load1\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"system\")\n |> filter(fn: (r) => r[\"_field\"] == \"load1\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\",\"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: true)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":14,"yTickStep":1},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":" e/s"}],"colorizeRows":true,"colors":[{"id":"xflqbsX-j3iq4ry5QOntK","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#8F8AF4"},{"id":"5H28HcITm6QVfQsXon0vq","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#A51414"},{"id":"25MrINwurNBkQqeKCkMPg","name":"Do Androids Dream of Electric Sheep?","type":"scale","hex":"#F4CF31"}],"geom":"line","height":4,"heightRatio":0.301556420233463,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Logstash EPS","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"in\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"out\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: -r._value}))\n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"in\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"logstash_events\")\n |> filter(fn: (r) => r[\"_field\"] == \"out\")\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\") \n |> map(fn: (r) => ({r with _value: -r._value}))\n |> group(columns: [\"_field\", \"host\", \"pipeline\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.301556420233463,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":18},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear"}],"colorizeRows":true,"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":0,"height":4,"hoverDimension":"auto","kind":"Single_Stat_Plus_Line","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Kafka Under Replicated Partitions","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_partition\")\n |> filter(fn: (r) => r[\"_field\"] == \"UnderReplicatedPartitions\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"partition\",\"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"kafka_partition\")\n |> filter(fn: (r) => r[\"_field\"] == \"UnderReplicatedPartitions\")\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"partition\",\"host\", \"role\"])\n |> yield(name: \"trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":22},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"UAehjIsi65P8u92M_3sQY","name":"Nineteen Eighty Four","type":"scale","hex":"#31C0F6"},{"id":"_SCP8Npp4NVMx2N4mfuzX","name":"Nineteen Eighty Four","type":"scale","hex":"#A500A5"},{"id":"BoMPg4R1KDp_UsRORdV3_","name":"Nineteen Eighty Four","type":"scale","hex":"#FF7E27"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"IO Wait","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_iowait\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"cpu\")\n |> filter(fn: (r) => r[\"cpu\"] == \"cpu-total\")\n |> filter(fn: (r) => r[\"_field\"] == \"usage_iowait\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":26},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"QDwChKZWuQV0BaJcEeSam","name":"Atlantis","type":"scale","hex":"#74D495"},{"id":"ThD0WTqKHltQEVlq9mo6K","name":"Atlantis","type":"scale","hex":"#3F3FBA"},{"id":"FBHYZiwDLKyQK3eRfUD-0","name":"Atlantis","type":"scale","hex":"#FF4D9E"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Swap Usage","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"swap\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"swap\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> roleFilter()\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":30},{"axes":[{"base":"10","name":"x","scale":"linear"},{"base":"10","name":"y","scale":"linear","suffix":"b/s"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"heightRatio":0.18482490272373542,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Monitor Interface Drops - Inbound","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop_in\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"drop_in\"}))"},{"query":"import \"join\"\n\nhostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nmanints = from(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"node_config\")\n |> hostFilter()\n |> filter(fn: (r) => r[\"_field\"] == \"monint\")\n |> distinct()\n |> group(columns: [\"host\"])\n\ntraffic = from(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"net\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop_in\")\n |> hostFilter()\n |> roleFilter()\n |> derivative(unit: 1s, nonNegative: true, columns: [\"_value\"], timeColumn: \"_time\")\n |> map(fn: (r) => ({r with \"_value\": r._value * 8.0}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"host\"])\n\njoin.inner(left: traffic, right: manints,\n on: (l,r) => l.interface == r._value,\n as: (l, r) => ({l with _value: l._value, result: \"Trend\"}))"}],"staticLegend":{"colorizeRows":true,"heightRatio":0.18482490272373542,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":34},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":" days"}],"colorizeRows":true,"colors":[{"id":"3PVw3hQuZUzyar7Js3mMH","name":"Ectoplasm","type":"scale","hex":"#DA6FF1"},{"id":"O34ux-D8Xq_1-eeWRyYYH","name":"Ectoplasm","type":"scale","hex":"#00717A"},{"id":"P04RoKOHBdLdvfrfFbn0F","name":"Ectoplasm","type":"scale","hex":"#ACFF76"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Stenographer PCAP Retention","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"pcapage\")\n |> filter(fn: (r) => r[\"_field\"] == \"seconds\")\n |> map(fn: (r) => ({ r with _value: r._value / (24.0 * 3600.0)}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])"},{"query":"import \"join\"\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"pcapage\")\n |> filter(fn: (r) => r[\"_field\"] == \"seconds\")\n |> map(fn: (r) => ({ r with _value: r._value / (24.0 * 3600.0)}))\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> group(columns: [\"_field\",\"host\"])\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":4,"widthRatio":1,"xCol":"_time","xPos":8,"yCol":"_value","yPos":46},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Suricata Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"suridrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":9},{"colors":[{"id":"0","name":"viridian","type":"min","hex":"#32B08C"},{"id":"5IArg2lDb8KvnphywgUXa","name":"pineapple","type":"threshold","hex":"#FFB94A","value":50},{"id":"yFhH3mtavjuAZh6cEt5lx","name":"fire","type":"threshold","hex":"#DC4E58","value":70},{"id":"1","name":"ruby","type":"max","hex":"#BF3D5E","value":100}],"decimalPlaces":0,"height":4,"kind":"Gauge","name":"Swap Usage","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"swap\")\n |> filter(fn: (r) => r[\"_field\"] == \"used_percent\")\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n: 1)\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"}],"staticLegend":{},"suffix":"%","tickSuffix":"%","width":3,"xPos":9,"yPos":2},{"colors":[{"id":"base","name":"white","type":"text","hex":"#ffffff"}],"fieldOptions":[{"displayName":"Host","fieldName":"host","visible":true},{"displayName":"Name","fieldName":"container_name","visible":true},{"displayName":"Status","fieldName":"container_status","visible":true},{"displayName":"OOM Killed","fieldName":"_value","visible":true},{"displayName":"_start","fieldName":"_start","visible":true},{"displayName":"_stop","fieldName":"_stop","visible":true},{"displayName":"_time","fieldName":"_time","visible":true},{"displayName":"_field","fieldName":"_field","visible":true},{"displayName":"_measurement","fieldName":"_measurement","visible":true},{"displayName":"engine_host","fieldName":"engine_host","visible":true},{"displayName":"role","fieldName":"role","visible":true},{"displayName":"server_version","fieldName":"server_version","visible":true},{"displayName":"container_image","fieldName":"container_image","visible":true},{"displayName":"container_version","fieldName":"container_version","visible":true},{"displayName":"description","fieldName":"description","visible":true},{"displayName":"maintainer","fieldName":"maintainer","visible":true},{"displayName":"io.k8s.description","fieldName":"io.k8s.description","visible":true},{"displayName":"io.k8s.display-name","fieldName":"io.k8s.display-name","visible":true},{"displayName":"license","fieldName":"license","visible":true},{"displayName":"name","fieldName":"name","visible":true},{"displayName":"org.label-schema.build-date","fieldName":"org.label-schema.build-date","visible":true},{"displayName":"org.label-schema.license","fieldName":"org.label-schema.license","visible":true},{"displayName":"org.label-schema.name","fieldName":"org.label-schema.name","visible":true},{"displayName":"org.label-schema.schema-version","fieldName":"org.label-schema.schema-version","visible":true},{"displayName":"org.label-schema.url","fieldName":"org.label-schema.url","visible":true},{"displayName":"org.label-schema.vcs-ref","fieldName":"org.label-schema.vcs-ref","visible":true},{"displayName":"org.label-schema.vcs-url","fieldName":"org.label-schema.vcs-url","visible":true},{"displayName":"org.label-schema.vendor","fieldName":"org.label-schema.vendor","visible":true},{"displayName":"org.label-schema.version","fieldName":"org.label-schema.version","visible":true},{"displayName":"org.opencontainers.image.created","fieldName":"org.opencontainers.image.created","visible":true},{"displayName":"org.opencontainers.image.licenses","fieldName":"org.opencontainers.image.licenses","visible":true},{"displayName":"org.opencontainers.image.title","fieldName":"org.opencontainers.image.title","visible":true},{"displayName":"org.opencontainers.image.vendor","fieldName":"org.opencontainers.image.vendor","visible":true},{"displayName":"release","fieldName":"release","visible":true},{"displayName":"summary","fieldName":"summary","visible":true},{"displayName":"url","fieldName":"url","visible":true},{"displayName":"vendor","fieldName":"vendor","visible":true},{"displayName":"version","fieldName":"version","visible":true},{"displayName":"org.label-schema.usage","fieldName":"org.label-schema.usage","visible":true},{"displayName":"org.opencontainers.image.documentation","fieldName":"org.opencontainers.image.documentation","visible":true},{"displayName":"org.opencontainers.image.revision","fieldName":"org.opencontainers.image.revision","visible":true},{"displayName":"org.opencontainers.image.source","fieldName":"org.opencontainers.image.source","visible":true},{"displayName":"org.opencontainers.image.url","fieldName":"org.opencontainers.image.url","visible":true},{"displayName":"org.opencontainers.image.version","fieldName":"org.opencontainers.image.version","visible":true},{"displayName":"org.opencontainers.image.description","fieldName":"org.opencontainers.image.description","visible":true}],"height":4,"kind":"Table","name":"Most Recent Container Events","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"docker_container_status\")\n |> filter(fn: (r) => r[\"_field\"] == \"oomkilled\")\n |> filter(fn: (r) => r[\"container_status\"] != \"running\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"container_name\", \"host\"])\n |> last()\n |> group()\n |> keep(columns: [\"_value\", \"container_name\", \"host\", \"container_status\"])"}],"staticLegend":{},"tableOptions":{"sortBy":"container_name","verticalTimeAxis":true},"timeFormat":"YYYY-MM-DD HH:mm:ss","width":3,"xPos":9,"yPos":6},{"axes":[{"base":"10","name":"x","scale":"linear"},{"name":"y","scale":"linear","suffix":"%"}],"colorizeRows":true,"colors":[{"id":"TtgHQAXNep94KBgtu48C_","name":"Cthulhu","type":"scale","hex":"#FDC44F"},{"id":"_IuzkORho_8QXTE6vMllv","name":"Cthulhu","type":"scale","hex":"#007C76"},{"id":"bUszW_YI_9oColDbLNQ-d","name":"Cthulhu","type":"scale","hex":"#8983FF"}],"geom":"line","height":4,"hoverDimension":"auto","kind":"Xy","legendColorizeRows":true,"legendOpacity":1,"legendOrientationThreshold":100000000,"name":"Zeek Capture Loss","opacity":1,"orientationThreshold":100000000,"position":"overlaid","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekcaptureloss\")\n |> filter(fn: (r) => r[\"_field\"] == \"loss\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"mean\")"},{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nroleFilter = (tables=<-) =>\n if v.Role != \"(All)\" then\n tables |> filter(fn: (r) => r[\"role\"] == v.Role)\n else\n tables\n\nfrom(bucket: \"telegraf/so_long_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"zeekcaptureloss\")\n |> filter(fn: (r) => r[\"_field\"] == \"loss\")\n |> hostFilter()\n |> roleFilter()\n |> group(columns: [\"_field\", \"host\", \"role\"])\n |> aggregateWindow(every: v.windowPeriod, fn: mean, createEmpty: false)\n |> yield(name: \"Trend\")"}],"staticLegend":{"colorizeRows":true,"opacity":1,"orientationThreshold":100000000,"widthRatio":1},"width":3,"widthRatio":1,"xCol":"_time","xPos":9,"yCol":"_value","yPos":42},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"Stenographer Loss","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n\nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"stenodrop\")\n |> filter(fn: (r) => r[\"_field\"] == \"drop\")\n |> map(fn: (r) => ({r with _value: r._value * 100.0}))\n |> hostFilter()\n |> group(columns: [\"host\"])\n |> last()\n |> aggregateWindow(every: v.windowPeriod, fn: mean)\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":"%","width":1,"xPos":10},{"colors":[{"id":"base","name":"laser","type":"text","hex":"#00C9FF"}],"decimalPlaces":1,"height":2,"kind":"Single_Stat","name":"PCAP Retention","queries":[{"query":"hostFilter = (tables=<-) =>\n if v.Host != \"(All)\" then\n tables |> filter(fn: (r) => r[\"host\"] == v.Host)\n else\n tables\n \nfrom(bucket: \"telegraf/so_short_term\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"pcapage\")\n |> filter(fn: (r) => r[\"_field\"] == \"seconds\")\n |> hostFilter()\n |> map(fn: (r) => ({r with _value: r._value / (24.0 * 60.0 * 60.0)}))\n |> group(columns: [\"host\"])\n |> last()\n |> highestMax(n:1)"}],"staticLegend":{},"suffix":" days","width":1,"xPos":11}],"description":"Visualize the Security Onion grid performance metrics and alarm statuses.","name":"Security Onion Performance"}}] \ No newline at end of file From d0565baaa32149985178f6e41b75aa1587551cc8 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Mon, 15 Jul 2024 14:43:47 -0600 Subject: [PATCH 63/80] New Config Values for Detections Bulk Indexer `maxScrollSize` defines the "page size" of each scroll request. `bulkIndexerWorkerCount` defines how many worker threads a bulk indexer should use. 0 or fewer indicates that 1 thread per CPU core should be used. --- salt/soc/defaults.yaml | 12 +++++++----- salt/soc/soc_soc.yaml | 4 ++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6ddf534c7..6d3667d0b 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1366,6 +1366,8 @@ soc: maxLogLength: 1024 asyncThreshold: 10 lookupTunnelParent: true + maxScrollSize: 10000 + bulkIndexerWorkerCount: -1 influxdb: hostUrl: token: @@ -2292,15 +2294,15 @@ soc: alert http $EXTERNAL_NET any -> $HOME_NET any (msg:"Example Rule Title - 'example' String Detected"; content:"example"; sid:[publicId]; rev:1;) strelka: | - /* + /* This is a YARA rule template. Replace all template values with your own values. The YARA rule name is the unique identifier for the rule. Docs: https://yara.readthedocs.io/en/stable/writingrules.html#writing-yara-rules - */ + */ rule Example // This identifier _must_ be unique { - meta: + meta: description = "Generic YARA Rule" author = "@SecurityOnion" date = "YYYY-MM-DD" @@ -2323,7 +2325,7 @@ soc: id: [publicId] status: 'experimental' description: | - This should be a detailed description of what this Detection focuses on: what we are trying to find and why we are trying to find it. + This should be a detailed description of what this Detection focuses on: what we are trying to find and why we are trying to find it. For example, from rule 97a80ec7-0e2f-4d05-9ef4-65760e634f6b: "Detects a whoami.exe executed with the /priv command line flag instructing the tool to show all current user privileges. This is often used after a privilege escalation attempt." references: - 'https://local.invalid' @@ -2332,7 +2334,7 @@ soc: tags: - detection.threat_hunting - attack.technique_id - logsource: + logsource: category: process_creation product: windows detection: diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 4975b75f0..35402f760 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -174,6 +174,10 @@ soc: lookupTunnelParent: description: When true, if a pivoted event appears to be encapsulated, such as in a VXLAN packet, then SOC will pivot to the VXLAN packet stream. When false, SOC will attempt to pivot to the encapsulated packet stream itself, but at the risk that it may be unable to locate it in the stored PCAP data. global: True + maxScrollSize: + description: The maximum number of documents to request in a single Elasticsearch scroll request. + bulkIndexWorkerCount: + description: The number of worker threads to use when bulk indexing data into Elasticsearch. A value below 1 will default to the number of CPUs available. sostatus: refreshIntervalMs: description: Duration (in milliseconds) between refreshes of the grid status. Shortening this duration may not have expected results, as the backend systems feeding this sostatus data will continue their updates as scheduled. From b814f32e0aab3742ac3f16776330763d18a242e9 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 16 Jul 2024 12:39:30 -0400 Subject: [PATCH 64/80] fix custom indices --- salt/elasticsearch/config.map.jinja | 5 +++++ salt/elasticsearch/template.map.jinja | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 8127df848..f6062e1f2 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -1,3 +1,8 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + {% from 'vars/globals.map.jinja' import GLOBALS %} {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS with context %} diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index b59c291a4..06041b537 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -1,3 +1,8 @@ +{# Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one + or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at + https://securityonion.net/license; you may not use this file except in compliance with the + Elastic License 2.0. #} + {% import_yaml 'elasticsearch/defaults.yaml' as ELASTICSEARCHDEFAULTS %} {% set DEFAULT_GLOBAL_OVERRIDES = ELASTICSEARCHDEFAULTS.elasticsearch.index_settings.pop('global_overrides') %} @@ -17,10 +22,14 @@ {% set ES_INDEX_SETTINGS = {} %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} -{# if policy isn't defined in the original index settings, then dont merge policy from the global_overrides #} -{# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} -{% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} -{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} +{# prevent this action from being performed on custom defined indices. #} +{# the custom defined index is not present in either of the dictionaries and fails to reder. #} +{% if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %} +{# dont merge policy from the global_overrides if policy isn't defined in the original index settingss #} +{# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} +{% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} +{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} +{% endif %} {% endif %} {% if settings.index_template is defined %} From 34c3a58efe651b6b04f5890f6d2d0b48494cb15e Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 16 Jul 2024 14:03:48 -0400 Subject: [PATCH 65/80] add cold policy --- salt/elasticsearch/defaults.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 36d673d70..0f6eb9a83 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -201,6 +201,11 @@ elasticsearch: order: desc policy: phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d hot: actions: rollover: @@ -237,6 +242,11 @@ elasticsearch: order: desc policy: phases: + cold: + actions: + set_priority: + priority: 0 + min_age: 60d hot: actions: rollover: From 03ca7977a0dc3227b6c6db11d6ac1c4a0e68dc48 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Tue, 16 Jul 2024 14:14:55 -0400 Subject: [PATCH 66/80] quote variables Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/manager/tools/sbin/soup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 371b447bb..3aa765c01 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -676,11 +676,11 @@ up_to_2.4.90() { kafkatrust=$(get_random_value) # rearranging the kafka pillar to reduce clutter in SOC UI kafkasavedpass=$(so-yaml.py get /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password) - kafkatrimpass=$(echo $kafkasavedpass | awk '{print $1}') + kafkatrimpass=$(echo "$kafkasavedpass" | awk '{print $1}') echo "Making changes to the Kafka pillar layout" so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password - so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password $kafkatrimpass - so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass $kafkatrust + so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass" + so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass "$kafkatrust" INSTALLEDVERSION=2.4.90 } From 72ad49ed122221c692069cce68e6f33546a48b85 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Tue, 16 Jul 2024 14:36:06 -0400 Subject: [PATCH 67/80] add policy for so-lists and so-items --- salt/elasticsearch/defaults.yaml | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 0f6eb9a83..be5f155b5 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -206,11 +206,23 @@ elasticsearch: set_priority: priority: 0 min_age: 60d + delete: + actions: + delete: {} + min_age: 365d hot: actions: rollover: - max_size: 50gb + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-lists: index_sorting: false index_template: @@ -247,11 +259,23 @@ elasticsearch: set_priority: priority: 0 min_age: 60d + delete: + actions: + delete: {} + min_age: 365d hot: actions: rollover: - max_size: 50gb + max_age: 30d + max_primary_shard_size: 50gb + set_priority: + priority: 100 min_age: 0ms + warm: + actions: + set_priority: + priority: 50 + min_age: 30d so-case: index_sorting: false index_template: From 6d18177f989d452e7e6707949ed58a79874b70b8 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 17 Jul 2024 10:16:11 -0400 Subject: [PATCH 68/80] only include global phases if defined in default for that index --- salt/elasticsearch/defaults.yaml | 38 ++------------------------- salt/elasticsearch/template.map.jinja | 12 +++++++++ 2 files changed, 14 insertions(+), 36 deletions(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index be5f155b5..36d673d70 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -201,28 +201,11 @@ elasticsearch: order: desc policy: phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d hot: actions: rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 + max_size: 50gb min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d so-lists: index_sorting: false index_template: @@ -254,28 +237,11 @@ elasticsearch: order: desc policy: phases: - cold: - actions: - set_priority: - priority: 0 - min_age: 60d - delete: - actions: - delete: {} - min_age: 365d hot: actions: rollover: - max_age: 30d - max_primary_shard_size: 50gb - set_priority: - priority: 100 + max_size: 50gb min_age: 0ms - warm: - actions: - set_priority: - priority: 50 - min_age: 30d so-case: index_sorting: false index_template: diff --git a/salt/elasticsearch/template.map.jinja b/salt/elasticsearch/template.map.jinja index 06041b537..507ea533d 100644 --- a/salt/elasticsearch/template.map.jinja +++ b/salt/elasticsearch/template.map.jinja @@ -22,14 +22,26 @@ {% set ES_INDEX_SETTINGS = {} %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.update(salt['defaults.merge'](ES_INDEX_SETTINGS_GLOBAL_OVERRIDES, ES_INDEX_PILLAR, in_place=False)) %} {% for index, settings in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES.items() %} + {# prevent this action from being performed on custom defined indices. #} {# the custom defined index is not present in either of the dictionaries and fails to reder. #} {% if index in ES_INDEX_SETTINGS_ORIG and index in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES %} + {# dont merge policy from the global_overrides if policy isn't defined in the original index settingss #} {# this will prevent so-elasticsearch-ilm-policy-load from trying to load policy on non ILM manged indices #} {% if not ES_INDEX_SETTINGS_ORIG[index].policy is defined and ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} {% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].pop('policy') %} {% endif %} + +{# this prevents and index from inderiting a policy phase from global overrides if it wasnt defined in the defaults. #} +{% if ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy is defined %} +{% for phase in ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.copy() %} +{% if ES_INDEX_SETTINGS_ORIG[index].policy.phases[phase] is not defined %} +{% do ES_INDEX_SETTINGS_GLOBAL_OVERRIDES[index].policy.phases.pop(phase) %} +{% endif %} +{% endfor %} +{% endif %} + {% endif %} {% if settings.index_template is defined %} From 4478d7b55a4f0a6e2381aec99fa9517055161821 Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Fri, 19 Jul 2024 09:32:47 -0400 Subject: [PATCH 69/80] kafka soup pillar fix Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/manager/tools/sbin/soup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 3aa765c01..566708d3c 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -676,7 +676,7 @@ up_to_2.4.90() { kafkatrust=$(get_random_value) # rearranging the kafka pillar to reduce clutter in SOC UI kafkasavedpass=$(so-yaml.py get /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password) - kafkatrimpass=$(echo "$kafkasavedpass" | awk '{print $1}') + kafkatrimpass=$(echo "$kafkasavedpass" | sed -n '1 p' ) echo "Making changes to the Kafka pillar layout" so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass" From 022df966c744bc780c8de281a3a31dbe92234030 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 18 Jul 2024 16:09:44 -0600 Subject: [PATCH 70/80] Remove Allow/Deny Regex, Add Suricata Enable/Disable Regex --- salt/soc/soc_soc.yaml | 38 ++++++++------------------------------ 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 35402f760..3732b1308 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -90,11 +90,6 @@ soc: helpLink: sigma.html forcedType: "[]string" multiline: True - allowRegex: - description: 'Regex used to filter imported Sigma rules. Deny regex takes precedence over the Allow regex setting.' - global: True - advanced: True - helpLink: sigma.html autoEnabledSigmaRules: default: &autoEnabledSigmaRules description: 'Sigma rules to automatically enable on initial import. Format is $Ruleset+$Level - for example, for the core community ruleset and critical level rules: core+critical. These will be applied based on role if defined and default if not.' @@ -103,11 +98,6 @@ soc: helpLink: sigma.html so-eval: *autoEnabledSigmaRules so-import: *autoEnabledSigmaRules - denyRegex: - description: 'Regex used to filter imported Sigma rules. Deny regex takes precedence over the Allow regex setting.' - global: True - advanced: True - helpLink: sigma.html communityRulesImportFrequencySeconds: description: 'How often to check for new Sigma rules (in seconds). This applies to both Community Rule Packages and any configured Git repos.' global: True @@ -199,21 +189,11 @@ soc: advanced: True forcedType: int strelkaengine: - allowRegex: - description: 'Regex used to filter imported YARA rules. Deny regex takes precedence over the Allow regex setting.' - global: True - advanced: True - helpLink: yara.html autoEnabledYaraRules: description: 'YARA rules to automatically enable on initial import. Format is $Ruleset - for example, for the default shipped ruleset: securityonion-yara' global: True advanced: True helpLink: sigma.html - denyRegex: - description: 'Regex used to filter imported YARA rules. Deny regex takes precedence over the Allow regex setting.' - global: True - advanced: True - helpLink: yara.html communityRulesImportFrequencySeconds: description: 'How often to check for new YARA rules (in seconds). This applies to both Community Rules and any configured Git repos.' global: True @@ -232,21 +212,19 @@ soc: helpLink: yara.html airgap: *serulesRepos suricataengine: - allowRegex: - description: 'Regex used to filter imported Suricata rules. Deny regex takes precedence over the Allow regex setting.' - global: True - advanced: True - helpLink: suricata.html - denyRegex: - description: 'Regex used to filter imported Suricata rules. Deny regex takes precedence over the Allow regex setting.' - global: True - advanced: True - helpLink: suricata.html communityRulesImportFrequencySeconds: description: 'How often to check for new Suricata rules (in seconds).' global: True advanced: True helpLink: suricata.html + disableRegex: + description: A list of regular expressions used to automatically disable rules that match any of them. Each regular expression is tested against the rule's content. + global: True + forcedType: "[]string" + enableRegex: + description: A list of regular expressions used to automatically enable rules that match any of them. Each regular expression is tested against the rule's content. + global: True + forcedType: "[]string" integrityCheckFrequencySeconds: description: 'How often the Suricata integrity checker runs (in seconds). This verifies the integrity of deployed rules.' global: True From 45b241317514e462e0badc79aa16679084c8edec Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Fri, 19 Jul 2024 12:45:24 -0600 Subject: [PATCH 71/80] Removed Allow/Deny Regexes, Added Enable/Disable Regex Update config and annotations for new regex support for suricata. --- salt/soc/defaults.yaml | 8 ++------ salt/soc/soc_soc.yaml | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/salt/soc/defaults.yaml b/salt/soc/defaults.yaml index 6d3667d0b..f33783507 100644 --- a/salt/soc/defaults.yaml +++ b/salt/soc/defaults.yaml @@ -1311,7 +1311,6 @@ soc: kratos: hostUrl: elastalertengine: - allowRegex: '' autoUpdateEnabled: true autoEnabledSigmaRules: default: @@ -1327,7 +1326,6 @@ soc: communityRulesImportFrequencySeconds: 86400 communityRulesImportErrorSeconds: 300 failAfterConsecutiveErrorCount: 10 - denyRegex: '' elastAlertRulesFolder: /opt/sensoroni/elastalert reposFolder: /opt/sensoroni/sigma/repos rulesFingerprintFile: /opt/sensoroni/fingerprints/sigma.fingerprint @@ -1392,7 +1390,6 @@ soc: userFiles: - rbac/users_roles strelkaengine: - allowRegex: '' autoEnabledYaraRules: - securityonion-yara autoUpdateEnabled: true @@ -1400,7 +1397,6 @@ soc: communityRulesImportErrorSeconds: 300 failAfterConsecutiveErrorCount: 10 compileYaraPythonScriptPath: /opt/sensoroni/yara/compile_yara.py - denyRegex: '' reposFolder: /opt/sensoroni/yara/repos rulesRepos: default: @@ -1415,14 +1411,14 @@ soc: stateFilePath: /opt/sensoroni/fingerprints/strelkaengine.state integrityCheckFrequencySeconds: 1200 suricataengine: - allowRegex: '' autoUpdateEnabled: true communityRulesImportFrequencySeconds: 86400 communityRulesImportErrorSeconds: 300 customRulesets: + disableRegex: [] + enableRegex: [] failAfterConsecutiveErrorCount: 10 communityRulesFile: /nsm/rules/suricata/emerging-all.rules - denyRegex: '' rulesFingerprintFile: /opt/sensoroni/fingerprints/emerging-all.fingerprint stateFilePath: /opt/sensoroni/fingerprints/suricataengine.state integrityCheckFrequencySeconds: 1200 diff --git a/salt/soc/soc_soc.yaml b/salt/soc/soc_soc.yaml index 3732b1308..b390c32c2 100644 --- a/salt/soc/soc_soc.yaml +++ b/salt/soc/soc_soc.yaml @@ -222,7 +222,7 @@ soc: global: True forcedType: "[]string" enableRegex: - description: A list of regular expressions used to automatically enable rules that match any of them. Each regular expression is tested against the rule's content. + description: A list of regular expressions used to automatically enable rules that match any of them. Each regular expression is tested against the rule's content. Takes priority over disableRegex matches. global: True forcedType: "[]string" integrityCheckFrequencySeconds: From cf1335dd84b83f03654037054fc7316995357d2a Mon Sep 17 00:00:00 2001 From: reyesj2 <94730068+reyesj2@users.noreply.github.com> Date: Sat, 20 Jul 2024 11:31:33 -0400 Subject: [PATCH 72/80] searchnode logstash-kafka cert generation Signed-off-by: reyesj2 <94730068+reyesj2@users.noreply.github.com> --- salt/kafka/init.sls | 5 +++++ salt/top.sls | 2 ++ 2 files changed, 7 insertions(+) diff --git a/salt/kafka/init.sls b/salt/kafka/init.sls index 0a9b3305f..038920cb8 100644 --- a/salt/kafka/init.sls +++ b/salt/kafka/init.sls @@ -18,7 +18,12 @@ include: - kafka.nodes {% endif %} {% if GLOBALS.pipeline == "KAFKA" and KAFKAMERGED.enabled %} +{% if grains.role in ['so-manager', 'so-managersearch', 'so-standalone', 'so-receiver'] %} - kafka.enabled +{# Searchnodes only run kafka.ssl state when Kafka is enabled #} +{% elif grains.role == "so-searchnode" %} + - kafka.ssl +{% endif %} {% else %} - kafka.disabled {% endif %} diff --git a/salt/top.sls b/salt/top.sls index 3bd3b2d34..7365e0e83 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -155,6 +155,7 @@ base: - nginx - elasticfleet.install_agent_grid - stig + - kafka '*_managersearch and G@saltversion:{{saltversion}}': - match: compound @@ -184,6 +185,7 @@ base: - utility - elasticfleet - stig + - kafka '*_heavynode and G@saltversion:{{saltversion}}': - match: compound From 7dfb75ba6b8cb531d72bc396f0b64ffd75feca16 Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 22 Jul 2024 11:02:56 -0400 Subject: [PATCH 73/80] remove unused test parameters from setup --- setup/so-functions | 26 ++------------------------ setup/so-setup | 6 ------ 2 files changed, 2 insertions(+), 30 deletions(-) diff --git a/setup/so-functions b/setup/so-functions index 02467117e..edeb8b494 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1253,34 +1253,12 @@ nginx_pillar() { soc_pillar() { title "Creating the SOC pillar" touch $adv_soc_pillar_file + added_module_line=0 printf '%s\n'\ "soc:"\ " config:"\ " server:"\ - " srvKey: '$SOCSRVKEY'"\ - " modules:"\ - " elastalertengine:"\ - " allowRegex: '$ELASTALERT_ALLOW_REGEX'" > "$soc_pillar_file" - if [[ -n "$ELASTALERT_FAIL_ERROR_COUNT" ]]; then - printf '%s\n'\ - " failAfterConsecutiveErrorCount: $ELASTALERT_FAIL_ERROR_COUNT" >> "$soc_pillar_file" - fi - - printf '%s\n'\ - " strelkaengine:"\ - " allowRegex: '$STRELKA_ALLOW_REGEX'" >> "$soc_pillar_file" - if [[ -n "$STRELKA_FAIL_ERROR_COUNT" ]]; then - printf '%s\n'\ - " failAfterConsecutiveErrorCount: $STRELKA_FAIL_ERROR_COUNT" >> "$soc_pillar_file" - fi - - printf '%s\n'\ - " suricataengine:"\ - " allowRegex: '$SURICATA_ALLOW_REGEX'" >> "$soc_pillar_file" - if [[ -n "$SURICATA_FAIL_ERROR_COUNT" ]]; then - printf '%s\n'\ - " failAfterConsecutiveErrorCount: $SURICATA_FAIL_ERROR_COUNT" >> "$soc_pillar_file" - fi + " srvKey: '$SOCSRVKEY'" > "$soc_pillar_file" if [[ $telemetry -ne 0 ]]; then echo " telemetryEnabled: false" >> $soc_pillar_file diff --git a/setup/so-setup b/setup/so-setup index 571fcc75d..bd8a8c6ba 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -245,12 +245,6 @@ if [ -n "$test_profile" ]; then WEBUSER=onionuser@somewhere.invalid WEBPASSWD1=0n10nus3r WEBPASSWD2=0n10nus3r - STRELKA_ALLOW_REGEX="EquationGroup_Toolset_Apr17__ELV_.*" - STRELKA_FAIL_ERROR_COUNT=1 - ELASTALERT_ALLOW_REGEX="Security Onion" - ELASTALERT_FAIL_ERROR_COUNT=1 - SURICATA_ALLOW_REGEX="(200033\\d|2100538|2102466)" - SURICATA_FAIL_ERROR_COUNT=1 update_sudoers_for_testing fi From 2e17e93cfe25898af720c0d51bbef54232a8a9bc Mon Sep 17 00:00:00 2001 From: Jason Ertel Date: Mon, 22 Jul 2024 11:04:45 -0400 Subject: [PATCH 74/80] remove unused test parameters from setup --- setup/so-functions | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/so-functions b/setup/so-functions index edeb8b494..4fe5ddebc 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -1253,7 +1253,6 @@ nginx_pillar() { soc_pillar() { title "Creating the SOC pillar" touch $adv_soc_pillar_file - added_module_line=0 printf '%s\n'\ "soc:"\ " config:"\ From 66b087f12fb2eb8536244bafa963a6aa813f6b66 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Thu, 25 Jul 2024 11:49:57 -0400 Subject: [PATCH 75/80] 2.4.90 --- DOWNLOAD_AND_VERIFY_ISO.md | 22 ++++++++++----------- sigs/securityonion-2.4.90-20240724.iso.sig | Bin 0 -> 566 bytes 2 files changed, 11 insertions(+), 11 deletions(-) create mode 100644 sigs/securityonion-2.4.90-20240724.iso.sig diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 9ef2f4665..1040e8ea8 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,17 +1,17 @@ -### 2.4.80-20240624 ISO image released on 2024/06/25 +### 2.4.90-20240724 ISO image released on 2024/07/25 ### Download and Verify -2.4.80-20240624 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso +2.4.90-20240724 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.90-20240724.iso -MD5: 139F9762E926F9CB3C4A9528A3752C31 -SHA1: BC6CA2C5F4ABC1A04E83A5CF8FFA6A53B1583CC9 -SHA256: 70E90845C84FFA30AD6CF21504634F57C273E7996CA72F7250428DDBAAC5B1BD +MD5: 4843E0E47D6827E3CA1640621703F5F9 +SHA1: 3907C8A7418267BF7891858FF17950D4417F8A04 +SHA256: F1FC0554DE181DDE34B5677AC62B26364735C83052F9160ED2723D9A7B0CB6C1 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.90-20240724.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.80-20240624.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.90-20240724.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.80-20240624.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.90-20240724.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.80-20240624.iso.sig securityonion-2.4.80-20240624.iso +gpg --verify securityonion-2.4.90-20240724.iso.sig securityonion-2.4.90-20240724.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Mon 24 Jun 2024 02:42:03 PM EDT using RSA key ID FE507013 +gpg: Signature made Wed 24 Jul 2024 02:30:34 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.4.90-20240724.iso.sig b/sigs/securityonion-2.4.90-20240724.iso.sig new file mode 100644 index 0000000000000000000000000000000000000000..3156b0f38f58fd767ea82427519e15c2ee8650f0 GIT binary patch literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0%oB|N&pH85PT3| zxBgIY6MH@n|465MNLysgvvE$C9=wyh;N_3({ud0mGkm#9UP`=&CH^3c_F14aw>6Q3 zM6f>hW@@eT`n4O$2G{1je|MQ8B+KpZ_Ig9-s()h;Iv9$0}5it=f>0X5-+{`3(V_~ zTovBmte>EOuTe@MwV0HeOxZ}14$9g9x~oF)SYbeN>Y1ZC#iAWvxo15asdJ`!PJL`b zz_-O#*@o_n1(ZY-MwM3&9l9P#?9jq{9}$FhC!Cs>EDN{kjmW+pZ031ZN3x2?&q|58 zEF8Fy_KiczHJNB7g6#<5k}W}iKPj|CtA>RYh81Sfc=O&nvr0a4?TVc7RfX|0|CP&4 zj}x|nNxcw_8x`KDpWL_{TtUI16NA{|JVo+Bm3D!Nx*v()yu}R0y{|voEVgaBpg^Xx z+E2;MdSTi?85}pO@bF!HR4ZIc!wdZn@(vNAFu1j* z Date: Thu, 25 Jul 2024 12:53:04 -0600 Subject: [PATCH 76/80] so-detection refresh_interval => 1s Speeds up the refresh_interval so bulk indexing a single rule does not wait 30s. --- salt/elasticsearch/defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 36d673d70..36f44ac07 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -296,7 +296,7 @@ elasticsearch: limit: 1500 number_of_replicas: 0 number_of_shards: 1 - refresh_interval: 30s + refresh_interval: 1s sort: field: '@timestamp' order: desc From ccf88fa62b4721df7760beb6c554eefd5bfc23f6 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 25 Jul 2024 13:44:22 -0600 Subject: [PATCH 77/80] Add step to soup to set refresh_interval during upgrade The so-detection index needs it's refresh_interval reset during an upgrade. If the index doesn't exist, the config change will set it correctly when it is created. --- salt/manager/tools/sbin/soup | 57 ++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 566708d3c..03085b93b 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -1,7 +1,7 @@ #!/bin/bash # Copyright Security Onion Solutions LLC and/or licensed to Security Onion Solutions LLC under one -# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at +# or more contributor license agreements. Licensed under the Elastic License 2.0 as shown at # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. @@ -30,7 +30,7 @@ check_err() { [[ $ERR_HANDLED == true ]] && exit $exit_code if [[ $exit_code -ne 0 ]]; then - + set +e systemctl_func "start" "$cron_service_name" systemctl_func "start" "salt-master" @@ -108,7 +108,7 @@ add_common() { } airgap_mounted() { - # Let's see if the ISO is already mounted. + # Let's see if the ISO is already mounted. if [[ -f /tmp/soagupdate/SecurityOnion/VERSION ]]; then echo "The ISO is already mounted" else @@ -116,8 +116,8 @@ airgap_mounted() { echo "This is airgap. Ask for a location." echo "" cat << EOF -In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided. -For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso. +In order for soup to proceed, the path to the downloaded Security Onion ISO file, or the path to the CD-ROM or equivalent device containing the ISO media must be provided. +For example, if you have copied the new Security Onion ISO file to your home directory, then the path might look like /home/myuser/securityonion-2.x.y.iso. Or, if you have burned the new ISO onto an optical disk then the path might look like /dev/cdrom. EOF @@ -134,7 +134,7 @@ EOF exit 0 else echo "ISO has been mounted!" - fi + fi elif [[ -f $ISOLOC/SecurityOnion/VERSION ]]; then ln -s $ISOLOC /tmp/soagupdate echo "Found the update content" @@ -149,7 +149,7 @@ EOF echo "Device has been mounted!" fi else - echo "Could not find Security Onion ISO content at ${ISOLOC}" + echo "Could not find Security Onion ISO content at ${ISOLOC}" echo "Ensure the path you entered is correct, and that you verify the ISO that you downloaded." exit 0 fi @@ -195,7 +195,7 @@ check_airgap() { UPDATE_DIR=/tmp/soagupdate/SecurityOnion AGDOCKER=/tmp/soagupdate/docker AGREPO=/tmp/soagupdate/minimal/Packages - else + else is_airgap=1 fi } @@ -407,11 +407,11 @@ preupgrade_changes() { postupgrade_changes() { # This function is to add any new pillar items if needed. echo "Running post upgrade processes." - + [[ "$POSTVERSION" == 2.4.2 ]] && post_to_2.4.3 [[ "$POSTVERSION" == 2.4.3 ]] && post_to_2.4.4 - [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5 - [[ "$POSTVERSION" == 2.4.5 ]] && post_to_2.4.10 + [[ "$POSTVERSION" == 2.4.4 ]] && post_to_2.4.5 + [[ "$POSTVERSION" == 2.4.5 ]] && post_to_2.4.10 [[ "$POSTVERSION" == 2.4.10 ]] && post_to_2.4.20 [[ "$POSTVERSION" == 2.4.20 ]] && post_to_2.4.30 [[ "$POSTVERSION" == 2.4.30 ]] && post_to_2.4.40 @@ -569,13 +569,13 @@ up_to_2.4.5() { up_to_2.4.10() { echo "Nothing to do for 2.4.10" - + INSTALLEDVERSION=2.4.10 } up_to_2.4.20() { echo "Nothing to do for 2.4.20" - + INSTALLEDVERSION=2.4.20 } @@ -628,7 +628,7 @@ up_to_2.4.50() { mkdir /opt/so/rules/nids/suri chown socore:socore /opt/so/rules/nids/suri mv -v /opt/so/rules/nids/*.rules /opt/so/rules/nids/suri/. - + echo "Adding /nsm/elastic-fleet/artifacts to file_roots in /etc/salt/master using so-yaml" so-yaml.py append /etc/salt/master file_roots.base /nsm/elastic-fleet/artifacts @@ -681,6 +681,7 @@ up_to_2.4.90() { so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass" so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass "$kafkatrust" + so-elasticsearch-query so-detection/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}' INSTALLEDVERSION=2.4.90 } @@ -714,7 +715,7 @@ Documentation: https://docs.securityonion.net/en/2.4/telemetry.html ASSIST_EOF echo -n "Continue the upgrade with SOC Telemetry enabled [Y/n]? " - + read -r input input=$(echo "${input,,}" | xargs echo -n) echo "" @@ -755,7 +756,7 @@ suricata_idstools_migration() { rsync -av /opt/so/rules/nids/suri/local.rules /nsm/backup/detections-migration/suricata/local-rules if [[ -f /opt/so/saltstack/local/salt/idstools/rules/local.rules ]]; then rsync -av /opt/so/saltstack/local/salt/idstools/rules/local.rules /nsm/backup/detections-migration/suricata/local-rules/local.rules.bak - fi + fi #Tell SOC to migrate mkdir -p /opt/so/conf/soc/migrations @@ -772,7 +773,7 @@ playbook_migration() { crontab -l | grep -v 'so-playbook-ruleupdate_cron' | crontab - if grep -A 1 'playbook:' /opt/so/saltstack/local/pillar/minions/* | grep -q 'enabled: True'; then - + # Check for active Elastalert rules active_rules_count=$(find /opt/so/rules/elastalert/playbook/ -type f \( -name "*.yaml" -o -name "*.yml" \) | wc -l) @@ -864,7 +865,7 @@ upgrade_space() { fi else echo "You have enough space for upgrade. Proceeding with soup." - fi + fi } unmount_update() { @@ -922,7 +923,7 @@ upgrade_check() { fi else is_hotfix=false - fi + fi } @@ -934,7 +935,7 @@ upgrade_check_salt() { echo "Salt needs to be upgraded to $NEWSALTVERSION." UPGRADESALT=1 fi -} +} upgrade_salt() { SALTUPGRADED=True @@ -1052,7 +1053,7 @@ apply_hotfix() { mv /etc/pki/managerssl.key /etc/pki/managerssl.key.old systemctl_func "start" "salt-minion" (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG" - fi + fi else echo "No actions required. ($INSTALLEDVERSION/$HOTFIXVERSION)" fi @@ -1081,7 +1082,7 @@ apply_hotfix() { main() { trap 'check_err $?' EXIT - + if [ -n "$BRANCH" ]; then echo "SOUP will use the $BRANCH branch." echo "" @@ -1265,7 +1266,7 @@ main() { echo "Waiting on the Salt Master service to be ready." check_salt_master_status || fail "Can't access salt master or it is not ready. Check $SOUP_LOG for details." set -e - + echo "Running a highstate to complete the Security Onion upgrade on this manager. This could take several minutes." (wait_for_salt_minion "$MINIONID" "5" '/dev/stdout' || fail "Salt minion was not running or ready.") 2>&1 | tee -a "$SOUP_LOG" highstate @@ -1316,9 +1317,9 @@ main() { if [[ $NUM_MINIONS -gt 1 ]]; then cat << EOF - - - + + + This appears to be a distributed deployment. Other nodes should update themselves at the next Salt highstate (typically within 15 minutes). Do not manually restart anything until you know that all the search/heavy nodes in your deployment are updated. This is especially important if you are using true clustering for Elasticsearch. Each minion is on a random 15 minute check-in period and things like network bandwidth can be a factor in how long the actual upgrade takes. If you have a heavy node on a slow link, it is going to take a while to get the containers to it. Depending on what changes happened between the versions, Elasticsearch might not be able to talk to said heavy node until the update is complete. @@ -1371,13 +1372,13 @@ while getopts ":b:f:y" opt; do echo "Cannot run soup in unattended mode. You must run soup manually to accept the Elastic License." exit 1 else - UNATTENDED=true + UNATTENDED=true fi ;; f ) ISOLOC="$OPTARG" ;; - \? ) + \? ) echo "Usage: soup [-b] [-y] [-f ]" exit 1 ;; From 0cb067f6f29781afd0f46059bf4b1681df86f6e7 Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 25 Jul 2024 14:01:10 -0600 Subject: [PATCH 78/80] Don't forget history Also update so-detectionhistory to have a refresh_interval of 1s. --- salt/manager/tools/sbin/soup | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 03085b93b..292604d07 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -682,6 +682,7 @@ up_to_2.4.90() { so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass" so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass "$kafkatrust" so-elasticsearch-query so-detection/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}' + so-elasticsearch-query so-detectionhistory/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}' INSTALLEDVERSION=2.4.90 } From d6af3aab6df070b702d84d69ef261620d76e977f Mon Sep 17 00:00:00 2001 From: Corey Ogburn Date: Thu, 25 Jul 2024 14:05:09 -0600 Subject: [PATCH 79/80] Use a wildcard instead of making 2 requests --- salt/manager/tools/sbin/soup | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/manager/tools/sbin/soup b/salt/manager/tools/sbin/soup index 292604d07..bc1306673 100755 --- a/salt/manager/tools/sbin/soup +++ b/salt/manager/tools/sbin/soup @@ -681,8 +681,7 @@ up_to_2.4.90() { so-yaml.py remove /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.password so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.password "$kafkatrimpass" so-yaml.py add /opt/so/saltstack/local/pillar/kafka/soc_kafka.sls kafka.config.trustpass "$kafkatrust" - so-elasticsearch-query so-detection/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}' - so-elasticsearch-query so-detectionhistory/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}' + so-elasticsearch-query so-detection*/_settings -X PUT -d '{"index":{"refresh_interval":"1s"}}' INSTALLEDVERSION=2.4.90 } From 3e248da14da9b9c17af409126f4a286733f538be Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Mon, 29 Jul 2024 11:37:42 -0400 Subject: [PATCH 80/80] 2.4.90 --- DOWNLOAD_AND_VERIFY_ISO.md | 22 ++++++++++----------- sigs/securityonion-2.4.90-20240724.iso.sig | Bin 566 -> 0 bytes sigs/securityonion-2.4.90-20240729.iso.sig | Bin 0 -> 566 bytes 3 files changed, 11 insertions(+), 11 deletions(-) delete mode 100644 sigs/securityonion-2.4.90-20240724.iso.sig create mode 100644 sigs/securityonion-2.4.90-20240729.iso.sig diff --git a/DOWNLOAD_AND_VERIFY_ISO.md b/DOWNLOAD_AND_VERIFY_ISO.md index 1040e8ea8..df8904e0a 100644 --- a/DOWNLOAD_AND_VERIFY_ISO.md +++ b/DOWNLOAD_AND_VERIFY_ISO.md @@ -1,17 +1,17 @@ -### 2.4.90-20240724 ISO image released on 2024/07/25 +### 2.4.90-20240729 ISO image released on 2024/07/29 ### Download and Verify -2.4.90-20240724 ISO image: -https://download.securityonion.net/file/securityonion/securityonion-2.4.90-20240724.iso +2.4.90-20240729 ISO image: +https://download.securityonion.net/file/securityonion/securityonion-2.4.90-20240729.iso -MD5: 4843E0E47D6827E3CA1640621703F5F9 -SHA1: 3907C8A7418267BF7891858FF17950D4417F8A04 -SHA256: F1FC0554DE181DDE34B5677AC62B26364735C83052F9160ED2723D9A7B0CB6C1 +MD5: 9A7714F5922EE555F08675D25E6237D5 +SHA1: D3B331452627DB716906BA9F3922574DFA3852DC +SHA256: 5B0CE32543944DBC50C4E906857384211E1BE83EF409619778F18FC62017E0E0 Signature for ISO image: -https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.90-20240724.iso.sig +https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.90-20240729.iso.sig Signing key: https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2.4/main/KEYS @@ -25,22 +25,22 @@ wget https://raw.githubusercontent.com/Security-Onion-Solutions/securityonion/2. Download the signature file for the ISO: ``` -wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.90-20240724.iso.sig +wget https://github.com/Security-Onion-Solutions/securityonion/raw/2.4/main/sigs/securityonion-2.4.90-20240729.iso.sig ``` Download the ISO image: ``` -wget https://download.securityonion.net/file/securityonion/securityonion-2.4.90-20240724.iso +wget https://download.securityonion.net/file/securityonion/securityonion-2.4.90-20240729.iso ``` Verify the downloaded ISO image using the signature file: ``` -gpg --verify securityonion-2.4.90-20240724.iso.sig securityonion-2.4.90-20240724.iso +gpg --verify securityonion-2.4.90-20240729.iso.sig securityonion-2.4.90-20240729.iso ``` The output should show "Good signature" and the Primary key fingerprint should match what's shown below: ``` -gpg: Signature made Wed 24 Jul 2024 02:30:34 PM EDT using RSA key ID FE507013 +gpg: Signature made Thu 25 Jul 2024 06:51:11 PM EDT using RSA key ID FE507013 gpg: Good signature from "Security Onion Solutions, LLC " gpg: WARNING: This key is not certified with a trusted signature! gpg: There is no indication that the signature belongs to the owner. diff --git a/sigs/securityonion-2.4.90-20240724.iso.sig b/sigs/securityonion-2.4.90-20240724.iso.sig deleted file mode 100644 index 3156b0f38f58fd767ea82427519e15c2ee8650f0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 566 zcmV-60?GY}0y6{v0SEvc79j-41gSkXz6^6dp_W8^5Ma0dP;e6k0%oB|N&pH85PT3| zxBgIY6MH@n|465MNLysgvvE$C9=wyh;N_3({ud0mGkm#9UP`=&CH^3c_F14aw>6Q3 zM6f>hW@@eT`n4O$2G{1je|MQ8B+KpZ_Ig9-s()h;Iv9$0}5it=f>0X5-+{`3(V_~ zTovBmte>EOuTe@MwV0HeOxZ}14$9g9x~oF)SYbeN>Y1ZC#iAWvxo15asdJ`!PJL`b zz_-O#*@o_n1(ZY-MwM3&9l9P#?9jq{9}$FhC!Cs>EDN{kjmW+pZ031ZN3x2?&q|58 zEF8Fy_KiczHJNB7g6#<5k}W}iKPj|CtA>RYh81Sfc=O&nvr0a4?TVc7RfX|0|CP&4 zj}x|nNxcw_8x`KDpWL_{TtUI16NA{|JVo+Bm3D!Nx*v()yu}R0y{|voEVgaBpg^Xx z+E2;MdSTi?85}pO@bF!HR4ZIc!wdZn@(vNAFu1j* zsW?TJoPx?PD6dPHVIm=|UwJj&|)Wy3oHVgvVX2;0k=Ll=_*j+Zm>x;f<~O zdXUOTo_Y*rYOn$pEK0VU5R$3F+uPtwT2NeE@Fx^)G^Ft``h;}1`;Yt+sBP{cQbz#x zy=Oo6)f_Vc6?ZwHJ&D5Lgm5rJQC!jAY3JZ+fvTu~<041iR20dQ%9YkT%3=aSRwJ6o z=0j3S74~%Eh)1DHVolu=R3Y^7b>8F|wRu$PE`)!5q9f*s;|yEJGsjDfNaMA=D|R?~ zgVeS0lPRX}9Ox;zF$dJN{C88={DDk!#P~{$9avzu_iMqQNs83E8VXV|)~pz^sNrc) z!UlN-kV1FR?kW<=!O)G`Mc=D)>Ns Em)#Ey_y7O^ literal 0 HcmV?d00001