From fab0dd2badbd6bbc5f80aabc4b78b85642d336b2 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Wed, 3 Nov 2021 13:25:42 -0400 Subject: [PATCH] add repeating es ingest panel for nontc --- salt/grafana/defaults.yaml | 16 +- ..._ingest_performance_nontc_graph.json.jinja | 807 ++++++++++++++++++ ...ch_ingest_performance_tc_graph.json.jinja} | 8 +- salt/grafana/templates/searchnode.json | 29 + 4 files changed, 845 insertions(+), 15 deletions(-) create mode 100644 salt/grafana/panels/elasticsearch_ingest_performance_nontc_graph.json.jinja rename salt/grafana/panels/{elasticsearch_index_performance_graph.json.jinja => elasticsearch_ingest_performance_tc_graph.json.jinja} (98%) create mode 100644 salt/grafana/templates/searchnode.json diff --git a/salt/grafana/defaults.yaml b/salt/grafana/defaults.yaml index f91a424ca..f3c9a52c6 100644 --- a/salt/grafana/defaults.yaml +++ b/salt/grafana/defaults.yaml @@ -2735,10 +2735,7 @@ grafana: title: 'Pipeline Overview' templating: list: - servername: - text: All - value: "$__all" - role: + searchnode: allValue: true multi: true hide: 0 @@ -2757,12 +2754,9 @@ grafana: y: 0 h: 8 w: 12 - - - - elasticsearch_index_performance_graph: + elasticsearch_ingest_performance_nontc_graph: gridPos: - x: 12 - y: 0 + x: 0 + y: 8 h: 8 - w: 12 + w: 24 diff --git a/salt/grafana/panels/elasticsearch_ingest_performance_nontc_graph.json.jinja b/salt/grafana/panels/elasticsearch_ingest_performance_nontc_graph.json.jinja new file mode 100644 index 000000000..a9ecb55b6 --- /dev/null +++ b/salt/grafana/panels/elasticsearch_ingest_performance_nontc_graph.json.jinja @@ -0,0 +1,807 @@ +{ + "id": 445549, + "gridPos": { + "x": {{ PANELS.elasticsearch_ingest_performance_nontc_graph.gridPos.x }}, + "y": {{ PANELS.elasticsearch_ingest_performance_nontc_graph.gridPos.y }}, + "w": {{ PANELS.elasticsearch_ingest_performance_nontc_graph.gridPos.w }}, + "h": {{ PANELS.elasticsearch_ingest_performance_nontc_graph.gridPos.h }} + }, + "type": "timeseries", + "title": "Elastic Ingest Performance - $searchnode", + "datasource": "InfluxDB", + "pluginVersion": "8.2.1", + "interval": "30s", + "options": { + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "max", + "mean" + ] + } + }, + "targets": [ + { + "alias": "community.id_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_community_id_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "conditionals_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_conditional_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "convert_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "queryType": "randomWalk", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_convert_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "data.index.name_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "queryType": "randomWalk", + "rawQuery": false, + "refId": "F", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_date_index_name_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "data_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "queryType": "randomWalk", + "rawQuery": false, + "refId": "G", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_date_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "dissect_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "queryType": "randomWalk", + "rawQuery": false, + "refId": "H", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_dissect_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "dot.expander_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "queryType": "randomWalk", + "rawQuery": false, + "refId": "I", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_dot_expander_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "geoip_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "queryType": "randomWalk", + "rawQuery": false, + "refId": "K", + "resultFormat": "time_series", + "select": [ + [ + { + "type": "field", + "params": [ + "ingest_processor_stats_geoip_time_in_millis" + ] + }, + { + "type": "last", + "params": [] + }, + { + "type": "non_negative_difference", + "params": [] + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "grok_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "queryType": "randomWalk", + "rawQuery": false, + "refId": "L", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_grok_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "json_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "queryType": "randomWalk", + "rawQuery": false, + "refId": "O", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_json_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "kv_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "queryType": "randomWalk", + "rawQuery": false, + "refId": "P", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_kv_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "lowercase_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "queryType": "randomWalk", + "rawQuery": false, + "refId": "Q", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_lowercase_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "remove_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "queryType": "randomWalk", + "rawQuery": false, + "refId": "R", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_remove_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "rename_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "queryType": "randomWalk", + "rawQuery": false, + "refId": "S", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_rename_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "script_time", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "queryType": "randomWalk", + "rawQuery": false, + "refId": "T", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_script_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + }, + { + "alias": "url_decodes", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "queryType": "randomWalk", + "rawQuery": false, + "refId": "U", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "ingest_processor_stats_user_agent_time_in_millis" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [], + "type": "non_negative_difference" + } + ] + ], + "tags": [ + { + "key": "host", + "operator": "=", + "value": "$searchnode" + } + ] + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "barAlignment": 0, + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "mappings": [], + "unit": "ms", + "decimals": 0 + }, + "overrides": [] + }, + "repeat": "searchnodes", + "repeatDirection": "v", + "description": "", + "timeFrom": null, + "timeShift": null, + "scopedVars": { + "searchnodes": { + "text": "jppce2390sea5955-1", + "value": "jppce2390sea5955-1", + "selected": false + } + } +} diff --git a/salt/grafana/panels/elasticsearch_index_performance_graph.json.jinja b/salt/grafana/panels/elasticsearch_ingest_performance_tc_graph.json.jinja similarity index 98% rename from salt/grafana/panels/elasticsearch_index_performance_graph.json.jinja rename to salt/grafana/panels/elasticsearch_ingest_performance_tc_graph.json.jinja index 202829f67..d94ff2759 100644 --- a/salt/grafana/panels/elasticsearch_index_performance_graph.json.jinja +++ b/salt/grafana/panels/elasticsearch_ingest_performance_tc_graph.json.jinja @@ -8,10 +8,10 @@ }, "fill": 1, "gridPos": { - "x": {{ PANELS.elasticsearch_index_performance_graph.gridPos.x }}, - "y": {{ PANELS.elasticsearch_index_performance_graph.gridPos.y }}, - "w": {{ PANELS.elasticsearch_index_performance_graph.gridPos.w }}, - "h": {{ PANELS.elasticsearch_index_performance_graph.gridPos.h }} + "x": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.x }}, + "y": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.y }}, + "w": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.w }}, + "h": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.h }} }, "id": 445548, "legend": { diff --git a/salt/grafana/templates/searchnode.json b/salt/grafana/templates/searchnode.json new file mode 100644 index 000000000..899454636 --- /dev/null +++ b/salt/grafana/templates/searchnode.json @@ -0,0 +1,29 @@ +{ + "allValue": null, + "current": { + "selected": true, + "tags": [], + "text": [{{ TEMPLATES.searchnode.get('text', DASHBOARD)| json }}], + "value": [{{ TEMPLATES.searchnode.get('value', DASHBOARD)| json }}] + }, + "datasource": "InfluxDB", + "definition": "show tag values with key=\"host\" WHERE (\"role\" = \"searchnode\")", + "description": null, + "error": null, + "hide": {{ TEMPLATES.searchnode.get('hide', 0)| json }}, + "includeAll": {{ TEMPLATES.searchnode.get('includeAll', true)| json }}, + "label": "Searchnode", + "multi": {{ TEMPLATES.searchnode.get('multi', true)| json }}, + "name": "searchnode", + "options": [], + "query": "show tag values with key=\"host\" WHERE (\"role\" = \"searchnode\")", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false +}