CSEC_PUBLIC/securityonion
1
0
Fork 0
mirror of https://github.com/Security-Onion-Solutions/securityonion.git synced 2025-12-06 17:22:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

add pipeline overview for true cluster

Browse Source
This commit is contained in:
m0duspwnens
2021-11-08 12:09:35 -05:00
parent 0b28e89f3c
commit d67e34dac4
6 changed files with 394 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
salt/grafana/defaults.yaml
View File

@@ -2772,3 +2772,34 @@ grafana:
y: 24
h: 8
w: 24
pipeline_overview_tc:
title: 'Pipeline Overview'
templating:
list:
cluster_name:
includeAll: true
multi: true
hide: 0
text: All
value: "$__all"
panels:
logstash_indexing_eps_in_tc_total_graph:
gridPos:
x: 12
y: 0
h: 8
w: 12
elasticsearch_ingest_performance_tc_graph:
gridPos:
x: 0
y: 16
h: 8
w: 24
elasticsearch_pipeline_time_tc_graph:
gridPos:
x: 0
y: 24
h: 8
w: 24

6
salt/grafana/init.sls
View File

@@ -17,7 +17,11 @@
{% if grains.role == 'so-eval' %}
{% do DASHBOARDS.append('eval') %}
{% else %}
{% do DASHBOARDS.append('pipeline_overview') %}
{% if not salt['pillar.get']('elasticsearch:true_cluster', False) %}
{% do DASHBOARDS.append('pipeline_overview_nontc') %}
{% else %}
{% do DASHBOARDS.append('pipeline_overview_tc') %}
{% fi %}
{# Grab a unique listing of nodetypes that exists so that we create only the needed dashboards #}
{% for dashboard in salt['cmd.shell']("ls /opt/so/saltstack/local/pillar/minions/|awk -F'_' {'print $2'}|awk -F'.' {'print $1'}").split() %}
{% if dashboard in ALLOWED_DASHBOARDS %}

92
salt/grafana/panels/elasticsearch_ingest_performance_tc_graph.json.jinja
View File

@@ -8,7 +8,7 @@
},
"fill": 1,
"gridPos": {
"x": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.x }},
"x": {{ PANELS.elasticsearch_ingest_performance_graph.gridPos.x }},
"y": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.y }},
"w": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.w }},
"h": {{ PANELS.elasticsearch_ingest_performance_tc_graph.gridPos.h }}
@@ -72,9 +72,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -115,9 +115,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -158,9 +158,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -178,7 +178,7 @@
"measurement": "elasticsearch_clusterstats_nodes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"queryType": "randomWalk",
"rawQuery": false,
"refId": "F",
@@ -203,9 +203,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -223,7 +223,7 @@
"measurement": "elasticsearch_clusterstats_nodes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"queryType": "randomWalk",
"rawQuery": false,
"refId": "G",
@@ -248,9 +248,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -268,7 +268,7 @@
"measurement": "elasticsearch_clusterstats_nodes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"queryType": "randomWalk",
"rawQuery": false,
"refId": "H",
@@ -293,9 +293,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -313,7 +313,7 @@
"measurement": "elasticsearch_clusterstats_nodes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"queryType": "randomWalk",
"rawQuery": false,
"refId": "I",
@@ -338,9 +338,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -358,7 +358,7 @@
"measurement": "elasticsearch_clusterstats_nodes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"queryType": "randomWalk",
"rawQuery": false,
"refId": "K",
@@ -383,9 +383,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -403,7 +403,7 @@
"measurement": "elasticsearch_clusterstats_nodes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"queryType": "randomWalk",
"rawQuery": false,
"refId": "L",
@@ -428,9 +428,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -448,7 +448,7 @@
"measurement": "elasticsearch_clusterstats_nodes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"queryType": "randomWalk",
"rawQuery": false,
"refId": "O",
@@ -473,9 +473,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -493,7 +493,7 @@
"measurement": "elasticsearch_clusterstats_nodes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"queryType": "randomWalk",
"rawQuery": false,
"refId": "P",
@@ -518,9 +518,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -538,7 +538,7 @@
"measurement": "elasticsearch_clusterstats_nodes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"queryType": "randomWalk",
"rawQuery": false,
"refId": "Q",
@@ -563,9 +563,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -583,7 +583,7 @@
"measurement": "elasticsearch_clusterstats_nodes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"queryType": "randomWalk",
"rawQuery": false,
"refId": "R",
@@ -608,9 +608,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -628,7 +628,7 @@
"measurement": "elasticsearch_clusterstats_nodes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"queryType": "randomWalk",
"rawQuery": false,
"refId": "S",
@@ -653,9 +653,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -673,7 +673,7 @@
"measurement": "elasticsearch_clusterstats_nodes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"queryType": "randomWalk",
"rawQuery": false,
"refId": "T",
@@ -698,9 +698,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
},
@@ -718,7 +718,7 @@
"measurement": "elasticsearch_clusterstats_nodes",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"role\" = 'manager') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"query": "SELECT non_negative_difference(mode(\"ingest_processor_stats_date_index_name_time_in_millis\")) FROM \"elasticsearch_clusterstats_nodes\" WHERE (\"cluster_name\" = '$cluster_name') AND $timeFilter GROUP BY time($__interval) fill(linear)",
"queryType": "randomWalk",
"rawQuery": false,
"refId": "U",
@@ -743,9 +743,9 @@
],
"tags": [
{
"key": "role",
"key": "cluster_name",
"operator": "=",
"value": "manager"
"value": "$cluster_name"
}
]
}

135
salt/grafana/panels/elasticsearch_pipeline_time_tc_graph.json.jinja Normal file
View File

@@ -0,0 +1,135 @@
{
"id": 445552,
"gridPos": {
"x": {{ PANELS.elasticsearch_pipeline_time_nontc_graph.gridPos.x }},
"y": {{ PANELS.elasticsearch_pipeline_time_nontc_graph.gridPos.y }},
"w": {{ PANELS.elasticsearch_pipeline_time_nontc_graph.gridPos.w }},
"h": {{ PANELS.elasticsearch_pipeline_time_nontc_graph.gridPos.h }}
},
"type": "timeseries",
"title": "Pipeline Time",
"datasource": "InfluxDB",
"interval": "30s",
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "linear",
"barAlignment": 0,
"lineWidth": 1,
"fillOpacity": 0,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"pointSize": 5,
"stacking": {
"mode": "none",
"group": "A"
},
"axisPlacement": "auto",
"axisLabel": "",
"scaleDistribution": {
"type": "linear"
},
"hideFrom": {
"tooltip": false,
"viz": false,
"legend": false
},
"thresholdsStyle": {
"mode": "off"
}
},
"color": {
"mode": "palette-classic"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"mappings": [],
"unit": "ms"
},
"overrides": []
},
"options": {
"tooltip": {
"mode": "multi"
},
"legend": {
"displayMode": "table",
"placement": "right",
"calcs": [
"max",
"mean"
]
}
},
"targets": [
{
"alias": "$tag_host",
"groupBy": [
{
"params": [
"$__interval"
],
"type": "time"
},
{
"params": [
"host"
],
"type": "tag"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "elasticsearch_clusterstats_nodes",
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"type": "field",
"params": [
"ingest_processor_stats_pipeline_time_in_millis"
]
},
{
"type": "mean",
"params": []
},
{
"type": "non_negative_difference",
"params": []
}
]
],
"tags": [
{
"key": "cluster_name",
"operator": "=",
"value": "$cluster_name"
}
]
}
]
}

152
salt/grafana/panels/logstash_indexing_eps_in_tc_total_graph.json.jinja Normal file
View File

@@ -0,0 +1,152 @@
{
"id": 69001,
"gridPos": {
"x": {{ PANELS.logstash_indexing_eps_in_tc_total_graph.gridPos.x }},
"y": {{ PANELS.logstash_indexing_eps_in_tc_total_graph.gridPos.y }},
"w": {{ PANELS.logstash_indexing_eps_in_tc_total_graph.gridPos.w }},
"h": {{ PANELS.logstash_indexing_eps_in_tc_total_graph.gridPos.h }}
},
"type": "timeseries",
"title": "Total Searchnode Indexing Events Per Second",
"transformations": [
{
"id": "calculateField",
"options": {
"alias": "Total EPS",
"mode": "reduceRow",
"reduce": {
"reducer": "sum"
},
"replaceFields": true
}
}
],
"datasource": "InfluxDB",
"pluginVersion": "8.2.1",
"interval": "30s",
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "linear",
"barAlignment": 0,
"lineWidth": 1,
"fillOpacity": 10,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"pointSize": 5,
"stacking": {
"mode": "none",
"group": "A"
},
"axisPlacement": "auto",
"axisLabel": "EPS",
"scaleDistribution": {
"type": "linear"
},
"hideFrom": {
"tooltip": false,
"viz": false,
"legend": false
},
"thresholdsStyle": {
"mode": "off"
}
},
"color": {
"mode": "fixed",
"fixedColor": "orange"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"mappings": [],
"unit": "short"
},
"overrides": []
},
"options": {
"tooltip": {
"mode": "single"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"max",
"mean",
"min"
]
}
},
"targets": [
{
"groupBy": [
{
"params": [
"$__interval"
],
"type": "time"
},
{
"params": [
"host"
],
"type": "tag"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"hide": false,
"measurement": "logstash_events",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT non_negative_derivative(mean(\"in\"), 1s) FROM \"logstash_events\" WHERE (\"cluster_name\" = \"$cluster_name\") AND $timeFilter GROUP BY time($__interval), \"host\" fill(null)",
"rawQuery": false,
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"type": "field",
"params": [
"in"
]
},
{
"type": "mean",
"params": []
},
{
"type": "non_negative_derivative",
"params": [
"1s"
]
}
]
],
"tags": [
{
"key": "cluster_name",
"operator": "=",
"value": "$cluster_name"
}
]
}
],
"description": "",
"maxDataPoints": null,
"timeFrom": null,
"timeShift": null
}

25
salt/grafana/templates/cluster_name.json Normal file
View File

@@ -0,0 +1,25 @@
{
"allValue": null,
"current": {
"selected": true,
"tags": [],
"text": [{{ TEMPLATES.cluster_name.get('text', DASHBOARD)| json }}],
"value": [{{ TEMPLATES.cluster_name.get('value', DASHBOARD)| json }}]
},
"datasource": "InfluxDB",
"definition": "show tag values with key=\"cluster_name\"",
"description": null,
"error": null,
"hide": {{ TEMPLATES.cluster_name.get('hide', 0)| json }},
"includeAll": {{ TEMPLATES.cluster_name.get('includeAll', true)| json }},
"label": "Cluster Name",
"multi": {{ TEMPLATES.cluster_name.get('multi', true)| json }},
"name": "cluster_name",
"options": [],
"query": "show tag values with key=\"cluster_name\"",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
}