From d67e34dac4b9fe442e781455f3cd048136185ad4 Mon Sep 17 00:00:00 2001 From: m0duspwnens Date: Mon, 8 Nov 2021 12:09:35 -0500 Subject: [PATCH] add pipeline overview for true cluster --- salt/grafana/defaults.yaml | 31 ++++ salt/grafana/init.sls | 6 +- ...rch_ingest_performance_tc_graph.json.jinja | 92 +++++------ ...icsearch_pipeline_time_tc_graph.json.jinja | 135 ++++++++++++++++ ..._indexing_eps_in_tc_total_graph.json.jinja | 152 ++++++++++++++++++ salt/grafana/templates/cluster_name.json | 25 +++ 6 files changed, 394 insertions(+), 47 deletions(-) create mode 100644 salt/grafana/panels/elasticsearch_pipeline_time_tc_graph.json.jinja create mode 100644 salt/grafana/panels/logstash_indexing_eps_in_tc_total_graph.json.jinja create mode 100644 salt/grafana/templates/cluster_name.json diff --git a/salt/grafana/defaults.yaml b/salt/grafana/defaults.yaml index fb698b705..47279bf6d 100644 --- a/salt/grafana/defaults.yaml +++ b/salt/grafana/defaults.yaml @@ -2772,3 +2772,34 @@ grafana: y: 24 h: 8 w: 24 + + + pipeline_overview_tc: + title: 'Pipeline Overview' + templating: + list: + cluster_name: + includeAll: true + multi: true + hide: 0 + text: All + value: "$__all" + panels: + logstash_indexing_eps_in_tc_total_graph: + gridPos: + x: 12 + y: 0 + h: 8 + w: 12 + elasticsearch_ingest_performance_tc_graph: + gridPos: + x: 0 + y: 16 + h: 8 + w: 24 + elasticsearch_pipeline_time_tc_graph: + gridPos: + x: 0 + y: 24 + h: 8 + w: 24 diff --git a/salt/grafana/init.sls b/salt/grafana/init.sls index 4a5a1dfe2..745b03028 100644 --- a/salt/grafana/init.sls +++ b/salt/grafana/init.sls @@ -17,7 +17,11 @@ {% if grains.role == 'so-eval' %} {% do DASHBOARDS.append('eval') %} {% else %} - {% do DASHBOARDS.append('pipeline_overview') %} + {% if not salt['pillar.get']('elasticsearch:true_cluster', False) %} + {% do DASHBOARDS.append('pipeline_overview_nontc') %} + {% else %} + {% do DASHBOARDS.append('pipeline_overview_tc') %} + {% fi %} {# Grab a unique listing of nodetypes that exists so that we create only the needed dashboards #} {% for dashboard in salt['cmd.shell']("ls /opt/so/saltstack/local/pillar/minions/|awk -F'_' {'print $2'}|awk -F'.' {'print $1'}").split() %} {% if dashboard in ALLOWED_DASHBOARDS %} diff --git a/salt/grafana/panels/elasticsearch_ingest_performance_tc_graph.json.jinja b/salt/grafana/panels/elasticsearch_ingest_performance_tc_graph.json.jinja index d94ff2759..d129382d1 100644 --- a/salt/grafana/panels/elasticsearch_ingest_performance_tc_graph.json.jinja +++ b/salt/grafana/panels/elasticsearch_ingest_performance_tc_graph.json.jinja @@ -8,7 +8,7 @@ }, "fill": 1, "gridPos": { - "x": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.x }}, + "x": {{ PANELS.elasticsearch_ingest_performance_graph.gridPos.x }}, "y": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.y }}, "w": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.w }}, "h": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.h }} @@ -72,9 +72,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -115,9 +115,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -158,9 +158,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -178,7 +178,7 @@ "measurement": "elasticsearch_clusterstats_nodes", "orderByTime": "ASC", "policy": "default", - "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)", "queryType": "randomWalk", "rawQuery": false, "refId": "F", @@ -203,9 +203,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -223,7 +223,7 @@ "measurement": "elasticsearch_clusterstats_nodes", "orderByTime": "ASC", "policy": "default", - "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)", "queryType": "randomWalk", "rawQuery": false, "refId": "G", @@ -248,9 +248,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -268,7 +268,7 @@ "measurement": "elasticsearch_clusterstats_nodes", "orderByTime": "ASC", "policy": "default", - "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)", "queryType": "randomWalk", "rawQuery": false, "refId": "H", @@ -293,9 +293,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -313,7 +313,7 @@ "measurement": "elasticsearch_clusterstats_nodes", "orderByTime": "ASC", "policy": "default", - "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)", "queryType": "randomWalk", "rawQuery": false, "refId": "I", @@ -338,9 +338,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -358,7 +358,7 @@ "measurement": "elasticsearch_clusterstats_nodes", "orderByTime": "ASC", "policy": "default", - "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)", "queryType": "randomWalk", "rawQuery": false, "refId": "K", @@ -383,9 +383,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -403,7 +403,7 @@ "measurement": "elasticsearch_clusterstats_nodes", "orderByTime": "ASC", "policy": "default", - "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)", "queryType": "randomWalk", "rawQuery": false, "refId": "L", @@ -428,9 +428,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -448,7 +448,7 @@ "measurement": "elasticsearch_clusterstats_nodes", "orderByTime": "ASC", "policy": "default", - "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)", "queryType": "randomWalk", "rawQuery": false, "refId": "O", @@ -473,9 +473,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -493,7 +493,7 @@ "measurement": "elasticsearch_clusterstats_nodes", "orderByTime": "ASC", "policy": "default", - "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)", "queryType": "randomWalk", "rawQuery": false, "refId": "P", @@ -518,9 +518,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -538,7 +538,7 @@ "measurement": "elasticsearch_clusterstats_nodes", "orderByTime": "ASC", "policy": "default", - "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)", "queryType": "randomWalk", "rawQuery": false, "refId": "Q", @@ -563,9 +563,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -583,7 +583,7 @@ "measurement": "elasticsearch_clusterstats_nodes", "orderByTime": "ASC", "policy": "default", - "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)", "queryType": "randomWalk", "rawQuery": false, "refId": "R", @@ -608,9 +608,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -628,7 +628,7 @@ "measurement": "elasticsearch_clusterstats_nodes", "orderByTime": "ASC", "policy": "default", - "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)", "queryType": "randomWalk", "rawQuery": false, "refId": "S", @@ -653,9 +653,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -673,7 +673,7 @@ "measurement": "elasticsearch_clusterstats_nodes", "orderByTime": "ASC", "policy": "default", - "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)", "queryType": "randomWalk", "rawQuery": false, "refId": "T", @@ -698,9 +698,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] }, @@ -718,7 +718,7 @@ "measurement": "elasticsearch_clusterstats_nodes", "orderByTime": "ASC", "policy": "default", - "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)", + "query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)", "queryType": "randomWalk", "rawQuery": false, "refId": "U", @@ -743,9 +743,9 @@ ], "tags": [ { - "key": "role", + "key": "cluster_name", "operator": "=", - "value": "manager" + "value": "$cluster_name" } ] } diff --git a/salt/grafana/panels/elasticsearch_pipeline_time_tc_graph.json.jinja b/salt/grafana/panels/elasticsearch_pipeline_time_tc_graph.json.jinja new file mode 100644 index 000000000..185649598 --- /dev/null +++ b/salt/grafana/panels/elasticsearch_pipeline_time_tc_graph.json.jinja @@ -0,0 +1,135 @@ +{ + "id": 445552, + "gridPos": { + "x": {{ PANELS.elasticsearch_pipeline_time_nontc_graph.gridPos.x }}, + "y": {{ PANELS.elasticsearch_pipeline_time_nontc_graph.gridPos.y }}, + "w": {{ PANELS.elasticsearch_pipeline_time_nontc_graph.gridPos.w }}, + "h": {{ PANELS.elasticsearch_pipeline_time_nontc_graph.gridPos.h }} + }, + "type": "timeseries", + "title": "Pipeline Time", + "datasource": "InfluxDB", + "interval": "30s", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "barAlignment": 0, + "lineWidth": 1, + "fillOpacity": 0, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [], + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "max", + "mean" + ] + } + }, + "targets": [ + { + "alias": "$tag_host", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "host" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "elasticsearch_clusterstats_nodes", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "type": "field", + "params": [ + "ingest_processor_stats_pipeline_time_in_millis" + ] + }, + { + "type": "mean", + "params": [] + }, + { + "type": "non_negative_difference", + "params": [] + } + ] + ], + "tags": [ + { + "key": "cluster_name", + "operator": "=", + "value": "$cluster_name" + } + ] + } + ] +} + + + + + + + diff --git a/salt/grafana/panels/logstash_indexing_eps_in_tc_total_graph.json.jinja b/salt/grafana/panels/logstash_indexing_eps_in_tc_total_graph.json.jinja new file mode 100644 index 000000000..89e1deef5 --- /dev/null +++ b/salt/grafana/panels/logstash_indexing_eps_in_tc_total_graph.json.jinja @@ -0,0 +1,152 @@ +{ + "id": 69001, + "gridPos": { + "x": {{ PANELS.logstash_indexing_eps_in_tc_total_graph.gridPos.x }}, + "y": {{ PANELS.logstash_indexing_eps_in_tc_total_graph.gridPos.y }}, + "w": {{ PANELS.logstash_indexing_eps_in_tc_total_graph.gridPos.w }}, + "h": {{ PANELS.logstash_indexing_eps_in_tc_total_graph.gridPos.h }} + }, + "type": "timeseries", + "title": "Total Searchnode Indexing Events Per Second", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "Total EPS", + "mode": "reduceRow", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "datasource": "InfluxDB", + "pluginVersion": "8.2.1", + "interval": "30s", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "barAlignment": 0, + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "EPS", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "color": { + "mode": "fixed", + "fixedColor": "orange" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "single" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "max", + "mean", + "min" + ] + } + }, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "host" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "logstash_events", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_derivative(mean(\"in\"), 1s) FROM \"logstash_events\" WHERE (\"cluster_name\" = \"$cluster_name\") AND $timeFilter GROUP BY time($__interval), \"host\" fill(null)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "type": "field", + "params": [ + "in" + ] + }, + { + "type": "mean", + "params": [] + }, + { + "type": "non_negative_derivative", + "params": [ + "1s" + ] + } + ] + ], + "tags": [ + { + "key": "cluster_name", + "operator": "=", + "value": "$cluster_name" + } + ] + } + ], + "description": "", + "maxDataPoints": null, + "timeFrom": null, + "timeShift": null +} diff --git a/salt/grafana/templates/cluster_name.json b/salt/grafana/templates/cluster_name.json new file mode 100644 index 000000000..bf6dbfb3f --- /dev/null +++ b/salt/grafana/templates/cluster_name.json @@ -0,0 +1,25 @@ +{ + "allValue": null, + "current": { + "selected": true, + "tags": [], + "text": [{{ TEMPLATES.cluster_name.get('text', DASHBOARD)| json }}], + "value": [{{ TEMPLATES.cluster_name.get('value', DASHBOARD)| json }}] + }, + "datasource": "InfluxDB", + "definition": "show tag values with key=\"cluster_name\"", + "description": null, + "error": null, + "hide": {{ TEMPLATES.cluster_name.get('hide', 0)| json }}, + "includeAll": {{ TEMPLATES.cluster_name.get('includeAll', true)| json }}, + "label": "Cluster Name", + "multi": {{ TEMPLATES.cluster_name.get('multi', true)| json }}, + "name": "cluster_name", + "options": [], + "query": "show tag values with key=\"cluster_name\"", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" +}