sendnrw/netbird
2
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2026-09-13 02:09:08 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
embed-multi-client-knobs
netbird/infrastructure_files/observability/grafana/dashboards/management-enterprise.json
T

8858 lines
279 KiB
JSON
Raw Permalink Blame History

{
"annotations": [
{
"kind": "AnnotationQuery",
"spec": {
"builtIn": true,
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"query": {
"datasource": {
"name": "-- Grafana --"
},
"group": "grafana",
"kind": "DataQuery",
"spec": {},
"version": "v0"
}
}
},
{
"kind": "AnnotationQuery",
"spec": {
"enable": true,
"hide": false,
"iconColor": "red",
"name": "Deployments",
"query": {
"datasource": {
"name": "grafana"
},
"group": "datasource",
"kind": "DataQuery",
"spec": {
"expr": "",
"interval": "",
"refId": "Anno",
"tags": [],
"type": "tags"
},
"version": "v0"
}
}
}
],
"cursorSync": "Crosshair",
"editable": true,
"elements": {
"panel-1": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"exemplar": false,
"expr": "histogram_quantile(0.99, sum by(le) (rate(management_grpc_updatechannel_queue_length_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))",
"format": "table",
"fullMetaSearch": false,
"includeNullMetadata": false,
"instant": false,
"legendFormat": "Connected grpc streams",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "p99 of the update-channel queue length: how many network map updates are waiting to be delivered to a peer. The channel buffer is 100 messages - approaching it means updates start getting dropped and peers fall behind.",
"id": 1,
"links": [],
"title": "UpdateChannel Queue length max",
"vizConfig": {
"group": "stat",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "blue",
"value": 0
},
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 50
}
]
},
"unit": "none"
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-10": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_total_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\", type=\"read\"}[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "HTTP read duration",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "p95 duration of read API requests (GET/OPTIONS), from the total request duration histogram tagged type=read.",
"id": 10,
"links": [],
"title": "Read HTTP API Requests p95 Duration (GET/OPTIONS)",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "orange",
"value": 200
}
]
},
"unit": "ms"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-11": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(environment) (rate(management_updatechannel_close_one_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "CloseOne",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum(rate(management_updatechannel_send_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "Send",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "B"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum(rate(management_updatechannel_create_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60)",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "Create",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "C"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "rate(management_updatechannel_get_all_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "GetAll",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "D"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "rate(management_updatechannel_haschannel_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "HasChannel",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "E"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "rate(management_updatechannel_close_multiple_channels_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "CloseMultiple",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "F"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Rate per minute of update-channel operations: close one, send update, create, get all connected peers, has-channel and close multiple. Together they show how much channel churn an instance is handling.",
"id": 11,
"links": [],
"title": "Update Channel operations",
"vizConfig": {
"group": "barchart",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 69,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 1,
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"fieldMinMax": false,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "reqpm"
},
"overrides": []
},
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 0.7,
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true,
"sortBy": "Mean",
"sortDesc": true
},
"orientation": "auto",
"showValue": "auto",
"stacking": "normal",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
},
"xTickLabelRotation": 0,
"xTickLabelSpacing": 100
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-12": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": true,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "rate(management_updatechannel_create_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\", closed=\"true\"}[$__rate_interval])",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{{host}}",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": true,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(environment) (rate(management_updatechannel_create_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{{host}}",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "B"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "__expr__"
},
"group": "__expr__",
"kind": "DataQuery",
"spec": {
"expression": "100-$A/$B*100",
"type": "math"
},
"version": "v0"
},
"refId": "C"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Tracks how often a new peer update channel replaces a channel that was still open (closed=true), i.e. peers reconnecting before management noticed the previous stream was gone. The plotted expression is 100 - recreated/total, so the line drops as recreations rise.",
"id": 12,
"links": [],
"title": "Percentage of Recreated channels",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"axisSoftMax": 100,
"axisSoftMin": 0,
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "percent"
},
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "C"
},
"properties": [
{
"id": "displayName",
"value": "Recreation"
}
]
}
]
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-13": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_close_one_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "CloseOne",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_send_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "Send",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "B"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_create_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "Create",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "C"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_get_all_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "GetAll",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "D"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_haschannel_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "CloseMultiple",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "E"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_close_multiple_channels_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "HasChannel",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "F"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "p95 duration of each update-channel operation, in microseconds. These are in-memory operations behind a lock, so growth here means lock contention rather than I/O.",
"id": 13,
"links": [],
"title": "Update Channel methods p95 Duration",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"fieldMinMax": false,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "µs"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-14": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"exemplar": false,
"expr": "sum(management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"})",
"format": "time_series",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "Connected grpc streams",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Total number of peers currently holding an open gRPC Sync stream across all management instances (management.grpc.connected.streams). A sudden drop means peers were disconnected and will come back as a Login/Sync reconnect storm.",
"id": 14,
"links": [],
"title": "Connected peers",
"vizConfig": {
"group": "stat",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "blue",
"value": 0
},
{
"color": "red",
"value": 0
},
{
"color": "#EAB839",
"value": 8000
},
{
"color": "green",
"value": 11000
}
]
},
"unit": "short"
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": true,
"textMode": "auto",
"wideLayout": true
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-15": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"expr": "management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"}",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "{{host}}",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "B"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"editorMode": "builder",
"expr": "sum by(instance) (management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"})",
"instant": false,
"legendFormat": "Total",
"range": true
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Open peer Sync streams over time, in total per instance (A) and as raw series (B). Steps down mark instance restarts or LB rebalancing; the ramp back up is the reconnect wave.",
"id": 15,
"links": [],
"title": "Connected peers historical",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "short"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-16": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_sync_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "GRPC Sync duration",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "p95 duration of the Sync gRPC request - the time management needs to authenticate the peer, build its first network map and open the update channel. Primary latency SLI for peer connects.",
"id": 16,
"links": [],
"title": "gRPC Sync Request p95 Duration",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 200
}
]
},
"unit": "ms"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-17": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_login_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "GRPC Login duration",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "p95 duration of the Login gRPC request (peer authentication plus initial configuration and relay credentials). Rises when the IdP, the store or account locking is slow.",
"id": 17,
"links": [],
"title": "gRPC Login Request p95 Duration",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 200
}
]
},
"unit": "ms"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-18": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(le) (increase(management_account_get_peer_network_map_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"format": "heatmap",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Distribution of the time taken to build the network map returned to a single peer, in milliseconds.",
"id": 18,
"links": [],
"title": "GetPeerNetworkMap Latency",
"vizConfig": {
"group": "heatmap",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Oranges",
"steps": 64
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-09
},
"legend": {
"placement": "bottom",
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"axisWidth": 60,
"decimals": 1,
"reverse": false,
"unit": "ms"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-19": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"exemplar": false,
"expr": "sum by(le) (increase(management_account_network_map_object_count_objects_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"format": "heatmap",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"interval": "",
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Distribution of the number of objects (peers, routes, firewall rules, DNS entries and so on) in the network maps sent to clients. Large maps make both calculation and serialization more expensive.",
"id": 19,
"links": [],
"title": "NetworkMap Objects",
"vizConfig": {
"group": "heatmap",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Oranges",
"steps": 64
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-09
},
"legend": {
"placement": "bottom",
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"axisWidth": 60,
"decimals": 0,
"reverse": false,
"unit": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-2": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "sum by(environment) (rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "login",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "sum by(environment) (rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "sync",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "B"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "sum by(environment) (rate(management_grpc_key_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "get key",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "C"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Rate of the three peer-facing gRPC calls: Login (authenticate, return initial config and relay credentials), Sync (open the update channel and receive network map updates) and GetServerKey (fetch the server WireGuard public key). Elevated Login/Sync rates usually mean peers are reconnecting in a loop.",
"id": 2,
"links": [],
"title": "gRPC Requests",
"vizConfig": {
"group": "barchart",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 51,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 1,
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "reqps"
},
"overrides": []
},
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": true,
"groupWidth": 0.7,
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"orientation": "auto",
"showValue": "auto",
"stacking": "normal",
"text": {},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
},
"xTickLabelRotation": 0,
"xTickLabelSpacing": 100
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-20": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(le) (increase(management_updatechannel_tosyncresponse_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"format": "heatmap",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Distribution of the time taken to convert a calculated network map into the gRPC SyncResponse sent to the peer, in microseconds.",
"id": 20,
"links": [],
"title": "To Sync Response",
"vizConfig": {
"group": "heatmap",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Oranges",
"steps": 64
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-09
},
"legend": {
"placement": "bottom",
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"axisWidth": 60,
"decimals": 1,
"reverse": false,
"unit": "µs"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-23": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum by(le) (increase(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"format": "heatmap",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Distribution of per-account p95 Sync durations - each sample is one account's p95, flushed every 60s. Shows whether slow Syncs are spread across the fleet or concentrated in a few accounts.",
"id": 23,
"links": [],
"title": "GRPC Sync Latency",
"vizConfig": {
"group": "heatmap",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.6,
"fill": "dark-red",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Purples",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-09
},
"legend": {
"placement": "bottom",
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"axisWidth": 60,
"decimals": 1,
"reverse": false,
"unit": "ms"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-24": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(le) (increase(management_grpc_login_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"format": "heatmap",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Distribution of Login gRPC request durations across all peers. The right tail is what a peer experiences on startup; samples beyond 7s are what the high-latency counters in the Debugging row track.",
"id": 24,
"links": [],
"title": "GRPC Login Latency",
"vizConfig": {
"group": "heatmap",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Purples",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-09
},
"legend": {
"placement": "bottom",
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"axisWidth": 60,
"decimals": 1,
"reverse": false,
"unit": "ms"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-25": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "{{host}}",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"editorMode": "code",
"expr": "sum by(environment) (rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"instant": false,
"legendFormat": "Total",
"range": true
},
"version": "v0"
},
"refId": "B"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Rate of Login requests, per instance (A) and in total (B). Login is the most expensive peer call (IdP plus store writes); a sustained high rate means clients are restarting or failing to hold their Sync stream.",
"id": 25,
"links": [],
"title": "gRPC Login Request rate",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 200
}
]
},
"unit": "reqps"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-26": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "{{host}}",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"editorMode": "code",
"expr": "sum by(environment) (rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"instant": false,
"legendFormat": "Total",
"range": true
},
"version": "v0"
},
"refId": "B"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Rate of Sync requests, per instance (A) and in total (B). Every Sync opens or re-opens a peer update channel, so a spike here is a reconnect storm and predicts CPU spent on network map calculation.",
"id": 26,
"links": [],
"title": "gRPC Sync Request rate",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 200
}
]
},
"unit": "reqps"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-27": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(le) (increase(management_grpc_updatechannel_queue_length_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"format": "heatmap",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Distribution of update-channel queue lengths across all peers. Mass moving to the right shows a growing backlog of undelivered network map updates.",
"id": 27,
"links": [],
"title": "Update Channel heat map",
"vizConfig": {
"group": "heatmap",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Greens",
"steps": 73
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-09
},
"legend": {
"placement": "bottom",
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"axisWidth": 60,
"decimals": 1,
"reverse": false,
"unit": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-29": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(endpoint) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval])*60) > 10",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "{{endpoint}} > 10",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(environment) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval])*60)",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "Total",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "B"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Successful (2xx) GET requests per minute broken down by endpoint, limited to endpoints above 10 rpm. Shows which read endpoints dashboard and API clients hit hardest.",
"id": 29,
"links": [],
"title": "HTTP Read request counter",
"vizConfig": {
"group": "barchart",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 0,
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"fieldMinMax": false,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 0.7,
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true,
"sortBy": "Mean",
"sortDesc": true
},
"orientation": "auto",
"showValue": "auto",
"stacking": "normal",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
},
"xTickLabelRotation": 0,
"xTickLabelSpacing": 100
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-30": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(endpoint) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60) > 1",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "{{endpoint}} > 1",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": true,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(environment) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60)",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "Total",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "B"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Successful (2xx) non-GET/OPTIONS requests per minute by endpoint, limited to endpoints above 1 rpm. Writes take account locks and fan out network map updates, so read this together with the Network Map row.",
"id": 30,
"links": [],
"title": "HTTP Write request counter",
"vizConfig": {
"group": "barchart",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 0,
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"fieldMinMax": false,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "reqpm"
},
"overrides": []
},
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 0.7,
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true,
"sortBy": "Last *",
"sortDesc": true
},
"orientation": "auto",
"showValue": "auto",
"stacking": "normal",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
},
"xTickLabelRotation": 0,
"xTickLabelSpacing": 100
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-31": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(le) (increase(management_store_persistence_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"format": "heatmap",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Distribution of how long it takes to save or delete an account in the store (management.store.persistence.duration.ms). Widening buckets point at database write pressure.",
"id": 31,
"links": [],
"title": "Store Persistence latency",
"vizConfig": {
"group": "heatmap",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.6,
"fill": "dark-red",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Reds",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-09
},
"legend": {
"placement": "bottom",
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"axisWidth": 60,
"decimals": 1,
"reverse": false,
"unit": "ms"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-32": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(le) (increase(management_store_transaction_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"format": "heatmap",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Distribution of store transaction execution time. Nearly every API and gRPC call runs through a transaction, so this is the first place to look when latency rises everywhere at once.",
"id": 32,
"links": [],
"title": "Store Transaction latency",
"vizConfig": {
"group": "heatmap",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.6,
"fill": "dark-red",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Reds",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-09
},
"legend": {
"placement": "bottom",
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"axisWidth": 60,
"decimals": 1,
"reverse": false,
"unit": "ms"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-33": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(environment) (rate(management_account_peer_meta_update_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "{{endpoint}} > 10",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Rate per minute of peers reporting changed metadata (OS, version, hostname, addresses). Each change persists the peer and can trigger an account peers update.",
"id": 33,
"links": [],
"title": "Peer meta updates counter",
"vizConfig": {
"group": "barchart",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 0,
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"fieldMinMax": false,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "reqpm"
},
"overrides": []
},
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 0.7,
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true,
"sortBy": "Mean",
"sortDesc": true
},
"orientation": "auto",
"showValue": "auto",
"stacking": "normal",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
},
"xTickLabelRotation": 0,
"xTickLabelSpacing": 100
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-34": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(environment) (rate(management_grpc_sync_request_blocked_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "GRPC Sync request rate",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Rate of Sync requests rejected because the peer is blocked (login expired, peer disabled or account restricted). A rising line means clients retrying without valid access.",
"id": 34,
"links": [],
"title": "gRPC Sync Request Blocked rate",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 0
}
]
},
"unit": "reqps"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-35": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(environment) (rate(management_grpc_login_request_blocked_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "GRPC Login request rate",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Rate of Login requests rejected because the peer is blocked. Persistent volume here is usually expired peer logins retrying in a loop.",
"id": 35,
"links": [],
"title": "gRPC Login Request Blocked rate",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 0
}
]
},
"unit": "reqps"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-36": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(le) (increase(management_account_update_account_peers_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"format": "heatmap",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Distribution of the time taken by an account peers update: preparing the data and pushing a fresh network map to every peer in the account, in milliseconds. Scales with account size.",
"id": 36,
"links": [],
"title": "UpdateAccountPeers Latency",
"vizConfig": {
"group": "heatmap",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Oranges",
"steps": 64
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-09
},
"legend": {
"placement": "bottom",
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"axisWidth": 60,
"decimals": 1,
"reverse": false,
"unit": "ms"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-37": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(le) (increase(management_updatechannel_calc_networkmap_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"format": "heatmap",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Distribution of the time taken to calculate one peer's network map, in milliseconds. This is the dominant CPU cost when many peers reconnect at once.",
"id": 37,
"links": [],
"title": "Network Map Calculation",
"vizConfig": {
"group": "heatmap",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Oranges",
"steps": 64
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-09
},
"legend": {
"placement": "bottom",
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"axisWidth": 60,
"decimals": 1,
"reverse": false,
"unit": "ms"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-39": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(le) (increase(management_updatechannel_calc_posturechecks_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"format": "heatmap",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Distribution of the time taken to evaluate a peer's posture checks while building its network map, in microseconds.",
"id": 39,
"links": [],
"title": "Posture Check Calculation",
"vizConfig": {
"group": "heatmap",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Oranges",
"steps": 64
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-09
},
"legend": {
"placement": "bottom",
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"axisWidth": 60,
"decimals": 1,
"reverse": false,
"unit": "µs"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-45": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"editorMode": "code",
"exemplar": false,
"expr": "sort_desc(\n topk(\n 10,\n sum by (account_id) (\n increase(management_grpc_login_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n )\n )\n)",
"format": "table",
"instant": true,
"legendFormat": "__auto",
"range": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": [
{
"group": "organize",
"kind": "Transformation",
"spec": {
"options": {
"excludeByName": {
"Time": true
},
"includeByName": {},
"indexByName": {},
"renameByName": {
"Time": ""
}
}
}
}
]
}
},
"description": "Top 10 accounts by number of Login requests that exceeded the 7s high-latency threshold.",
"id": 45,
"links": [],
"title": "High Login Latency Ranking (login count per account)",
"vizConfig": {
"group": "table",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"filterable": false,
"footer": {
"reducers": []
},
"inspect": false,
"tooltip": {
"placement": "auto"
},
"wrapHeaderText": false
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"options": {
"cellHeight": "sm",
"showHeader": true,
"sortBy": [
{
"desc": false,
"displayName": "Time"
}
]
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-46": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"editorMode": "code",
"exemplar": false,
"expr": "sort_desc(\n topk(\n 10,\n sum by (account_id) (\n increase(management_grpc_sync_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n )\n )\n)",
"format": "table",
"instant": true,
"legendFormat": "__auto",
"range": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": [
{
"group": "organize",
"kind": "Transformation",
"spec": {
"options": {
"excludeByName": {
"Time": true
},
"includeByName": {},
"indexByName": {},
"renameByName": {
"Time": ""
}
}
}
}
]
}
},
"description": "Top 10 accounts by number of Sync requests that exceeded the 7s high-latency threshold. Use it to find which tenant is behind a latency spike.",
"id": 46,
"links": [],
"title": "High Sync Latency Ranking (sync count per account)",
"vizConfig": {
"group": "table",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"filterable": false,
"footer": {
"reducers": []
},
"inspect": false,
"tooltip": {
"placement": "auto"
},
"wrapHeaderText": false
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"options": {
"cellHeight": "sm",
"showHeader": true,
"sortBy": [
{
"desc": false,
"displayName": "Time"
}
]
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-47": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"editorMode": "code",
"expr": "count(\n sum by (account_id) (\n increase(management_grpc_sync_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n ) > 1\n)",
"legendFormat": "__auto",
"range": true
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Number of distinct accounts with more than one slow (>7s) Sync in the interval - tells you whether a latency spike is fleet-wide or limited to a few tenants.",
"id": 47,
"links": [],
"title": "Accounts with high Sync latency",
"vizConfig": {
"group": "gauge",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"options": {
"barShape": "flat",
"barWidthFactor": 0.5,
"effects": {
"barGlow": false,
"centerGlow": false,
"gradient": false
},
"endpointMarker": "point",
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"segmentCount": 1,
"segmentSpacing": 0.3,
"shape": "gauge",
"showThresholdLabels": false,
"showThresholdMarkers": true,
"sizing": "auto",
"sparkline": false,
"textMode": "auto"
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-48": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"editorMode": "code",
"expr": "count(\n sum by (account_id) (\n increase(management_grpc_login_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n ) > 1\n)",
"legendFormat": "__auto",
"range": true
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Number of distinct accounts with more than one slow (>7s) Login in the interval.",
"id": 48,
"links": [],
"title": "Accounts with high Login latency",
"vizConfig": {
"group": "gauge",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"options": {
"barShape": "flat",
"barWidthFactor": 0.5,
"effects": {
"barGlow": false,
"centerGlow": false,
"gradient": false
},
"endpointMarker": "point",
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"segmentCount": 1,
"segmentSpacing": 0.3,
"shape": "gauge",
"showThresholdLabels": false,
"showThresholdMarkers": true,
"sizing": "auto",
"sparkline": false,
"textMode": "auto"
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-50": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${logs_datasource}"
},
"group": "loki",
"kind": "DataQuery",
"spec": {
"direction": "backward",
"editorMode": "code",
"expr": "sort_desc(\ntopk(\n 20,\n sum by (peerID) (\n count_over_time(\n {job=\"$log_job\"} |= \"Sync took\" | regexp `peerID:\\s(?P<peerID>[^,\\]]+)` [$__range]\n )\n )\n)\n)",
"queryType": "instant"
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": [
{
"group": "organize",
"kind": "Transformation",
"spec": {
"options": {
"excludeByName": {
"Time": true
},
"includeByName": {},
"indexByName": {},
"renameByName": {
"Value #A": "number of times ",
"peerID": "peerPubKey"
}
}
}
}
]
}
},
"description": "Top 20 peers by number of Sync log lines in the selected range, parsed out of the management logs to find individual peers stuck in a reconnect loop. Pick the log stream with the `log stream` variable; the query matches the `Sync took` debug line, so management must run at debug level for this panel to have data.",
"id": 50,
"links": [],
"title": "Highest Syncs by peer key",
"vizConfig": {
"group": "table",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"filterable": false,
"footer": {
"reducers": []
},
"inspect": false,
"tooltip": {
"placement": "auto"
},
"wrapHeaderText": false
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "labels"
},
"properties": [
{
"id": "custom.width",
"value": 180
}
]
},
{
"matcher": {
"id": "byName",
"options": "peerPubKey"
},
"properties": [
{
"id": "custom.width",
"value": 455
}
]
}
]
},
"options": {
"cellHeight": "sm",
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "number of times "
}
]
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-52": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"editorMode": "builder",
"expr": "wsproxy_active_connections{job=~\"$job\", instance=~\"$instance\"}",
"legendFormat": "{{__name__}}",
"range": true
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Currently open WebSocket proxy connections. The wsproxy fronts gRPC for browser-based clients, so this is the number of connected web clients.",
"id": 52,
"links": [],
"title": "Proxy Active Connections",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-53": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"editorMode": "builder",
"expr": "delta(wsproxy_bytes_transferred_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])",
"legendFormat": "{{direction}}",
"range": true
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Bytes transferred through the WebSocket proxy per interval, split by direction.",
"id": 53,
"links": [],
"title": "Proxy Traffic by Direction",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "decbytes"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-54": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"editorMode": "builder",
"expr": "increase(wsproxy_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])",
"legendFormat": "{{error_type}}",
"range": true
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "WebSocket proxy errors grouped by error type - dial failures, read/write errors and protocol issues on browser client connections.",
"id": 54,
"links": [],
"title": "Errors by Type",
"vizConfig": {
"group": "barchart",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 1,
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 0.7,
"legend": {
"calcs": [],
"displayMode": "list",
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"orientation": "auto",
"showValue": "auto",
"stacking": "none",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
},
"xTickLabelRotation": 0,
"xTickLabelSpacing": 0
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-65": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"editorMode": "builder",
"exemplar": false,
"expr": "management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"}",
"instant": true,
"legendFormat": "{{host}}",
"range": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Open peer Sync streams per management instance. Use it to confirm the load balancer spreads streams evenly - a skewed instance carries proportionally more update-channel and network map work.",
"id": 65,
"links": [],
"title": "Connected Peers per Server",
"vizConfig": {
"group": "gauge",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"options": {
"barShape": "flat",
"barWidthFactor": 0.5,
"effects": {
"barGlow": false,
"centerGlow": false,
"gradient": false
},
"endpointMarker": "point",
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"segmentCount": 1,
"segmentSpacing": 0.3,
"shape": "gauge",
"showThresholdLabels": false,
"showThresholdMarkers": true,
"sizing": "auto",
"sparkline": false,
"textMode": "auto"
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-66": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(host) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval]) * 60)",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "{{host}}",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "B"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Successful GET requests per minute grouped by management instance - read load distribution across the fleet.",
"id": 66,
"links": [],
"title": "HTTP Read request counter",
"vizConfig": {
"group": "barchart",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 0,
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"fieldMinMax": false,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 0.7,
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true,
"sortBy": "Mean",
"sortDesc": true
},
"orientation": "auto",
"showValue": "auto",
"stacking": "normal",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
},
"xTickLabelRotation": 0,
"xTickLabelSpacing": 100
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-67": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(host) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60)",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "{{host}}",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "B"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Successful write requests per minute grouped by management instance - write load distribution across the fleet.",
"id": 67,
"links": [],
"title": "HTTP Write request counter",
"vizConfig": {
"group": "barchart",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 0,
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"fieldMinMax": false,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "reqpm"
},
"overrides": []
},
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 0.7,
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true,
"sortBy": "Last *",
"sortDesc": true
},
"orientation": "auto",
"showValue": "auto",
"stacking": "normal",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
},
"xTickLabelRotation": 0,
"xTickLabelSpacing": 100
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-7": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_total_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\", type=\"write\"}[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "HTTP write duration",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "p95 duration of write API requests (PUT/POST/DELETE), tagged type=write. Writes acquire account locks and trigger peer updates, so they are normally slower than reads.",
"id": 7,
"links": [],
"title": "Write HTTP API Requests p95 Duration (PUT/POST/DELETE)",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 200
}
]
},
"unit": "ms"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-74": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(le) (increase(management_pat_usage_distribution_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"format": "heatmap",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Distribution of personal access token usage counts, sampled once per minute per token. Mass in the high buckets means a single PAT is hammering the API.",
"id": 74,
"links": [],
"title": "PAT usage counter",
"vizConfig": {
"group": "heatmap",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Purples",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-09
},
"legend": {
"placement": "bottom",
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"axisWidth": 60,
"decimals": 1,
"reverse": false,
"unit": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-76": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "gnatsd_connz_subscriptions{job=~\"$job\", instance=~\"$instance\"}",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": true,
"legendFormat": "{{host}}",
"range": false,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Current subscriptions held by each client connected to NATS (connz). Signal instances subscribe per connected peer, so this tracks how peers are spread across signal nodes.",
"id": 76,
"links": [],
"title": "Subscriptions per node",
"vizConfig": {
"group": "stat",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-77": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "gnatsd_connz_subscriptions{job=~\"$job\", instance=~\"$instance\"}",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"legendFormat": "{{host}}",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Subscriptions per connected NATS client over time. Steps mark signal instances restarting and re-subscribing their peers.",
"id": 77,
"links": [],
"title": "Subscriptions per node",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-78": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"expr": "gnatsd_connz_num_connections{job=~\"$job\", instance=~\"$instance\"}",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "{{host}}",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Number of client connections on each NATS node - effectively how many signal instances are attached to that node.",
"id": 78,
"links": [],
"title": "Signal nodes connected per NATs",
"vizConfig": {
"group": "stat",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-79": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum by(environment) (rate(gnatsd_connz_out_msgs{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Rate of messages NATS delivers out to its clients: the peer signalling throughput flowing through the cluster.",
"id": 79,
"links": [],
"title": "Messages per Second",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "mps"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-8": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "HTTP request duration",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "p95 duration of incoming REST API requests by endpoint and method (management.http.request.duration.ms). Covers dashboard and API clients only - peer traffic is on the gRPC panels.",
"id": 8,
"links": [],
"title": "HTTP API Requests p95 Duration",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": true,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 200
}
]
},
"unit": "ms"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-80": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum by(environment) (rate(gnatsd_connz_out_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Rate of bytes NATS delivers out to its clients - signalling bandwidth, useful next to the message rate to spot unusually large payloads.",
"id": 80,
"links": [],
"title": "Data per Second",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "bytes"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-81": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"expr": "sum by(environment) (increase(gnatsd_varz_slow_consumers{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Increase of the NATS slow-consumer counter. Anything above zero means a signal node could not read fast enough and NATS dropped its connection, so peers lose signalling until it reconnects.",
"id": 81,
"links": [],
"title": "Slow consumers detected",
"vizConfig": {
"group": "barchart",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 1,
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "none"
},
"overrides": []
},
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 0.7,
"legend": {
"calcs": [],
"displayMode": "list",
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"orientation": "auto",
"showValue": "auto",
"stacking": "none",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
},
"xTickLabelRotation": 0,
"xTickLabelSpacing": 100
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-82": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"expr": "gnatsd_varz_subscriptions{job=~\"$job\", instance=~\"$instance\"}",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "{{host}}",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Total subscriptions registered on each NATS server (varz). Grows roughly with the number of connected peers.",
"id": 82,
"links": [],
"title": "Total number of NATS subscriptions",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-83": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "{{instance}}",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Resident memory of the scraped processes. process_resident_memory_bytes is exposed by every Go/Prometheus process, so narrow the job and instance variables to the NATS nodes to read this panel as NATS memory.",
"id": 83,
"links": [],
"title": "NATS memory consumption",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "bytes"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-84": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "idelta(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[2m])",
"format": "heatmap",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Per-account Sync p95 samples as a heatmap (2m deltas). Use it to spot a handful of large or slow accounts pulling the overall Sync latency up.",
"id": 84,
"links": [],
"title": "GRPC Sync Latency By Account",
"vizConfig": {
"group": "heatmap",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.6,
"fill": "dark-red",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Purples",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-09
},
"legend": {
"placement": "bottom",
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"axisWidth": 60,
"decimals": 1,
"reverse": false,
"unit": "ms"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-85": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "idelta(management_grpc_login_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[2m])",
"format": "heatmap",
"fullMetaSearch": false,
"includeNullMetadata": true,
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Per-account Login p95 samples as a heatmap (2m deltas). Highlights accounts whose logins are much slower than the fleet average.",
"id": 85,
"links": [],
"title": "GRPC Login Latency By Account",
"vizConfig": {
"group": "heatmap",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "dark-orange",
"mode": "scheme",
"reverse": false,
"scale": "exponential",
"scheme": "Purples",
"steps": 128
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-09
},
"legend": {
"placement": "bottom",
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"axisWidth": 60,
"decimals": 1,
"reverse": false,
"unit": "ms"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-86": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "GRPC Sync duration",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "p95 of the per-account Sync p95 distribution - the latency seen by the worst-served accounts rather than by the average request. Diverges from the global p95 when only a few accounts are hurting.",
"id": 86,
"links": [],
"title": "gRPC Sync Request p95 Duration (by Account)",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 500
}
]
},
"unit": "ms"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-87": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_login_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "GRPC Login duration",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "p95 of the per-account Login p95 distribution - login latency for the worst-served accounts rather than the average request.",
"id": 87,
"links": [],
"title": "gRPC Login Request p95 Duration (by Account)",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 32,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "line"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 200
}
]
},
"unit": "ms"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-88": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "sum by(environment) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2",
"format": "time_series",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"interval": "",
"legendFormat": "{{environment}}",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Account peers updates triggered per minute (2m increase halved). Each trigger recalculates and pushes network maps to every peer in the account.",
"id": 88,
"links": [],
"title": "NetworkMap Triggers Total",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-BlYlRd",
"seriesBy": "last"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 19,
"gradientMode": "scheme",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "cpm"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-89": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"exemplar": false,
"expr": "sum by(resource) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m]))",
"format": "time_series",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"interval": "",
"legendFormat": "__auto",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Account peers updates over 2m intervals, grouped by the resource that triggered them - the time series behind the pie chart above.",
"id": 89,
"links": [],
"title": "NetworkMap Triggers By Source",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic",
"seriesBy": "max"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.9,
"drawStyle": "line",
"fillOpacity": 50,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 2,
"pointSize": 1,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "always",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "cpm"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-9": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(environment) (rate(management_idp_authenticate_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "authenticate (mgtm->auth0)",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "rate(management_idp_update_user_meta_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$interval])*60",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "update user meta",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "B"
}
},
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"expr": "rate(management_idp_get_account_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60",
"fullMetaSearch": false,
"includeNullMetadata": false,
"legendFormat": "get account",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "C"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Requests per minute that management sends to the configured identity provider: authenticate (service token refresh), user metadata updates and account lookups. IdP throttling shows up here before it shows up as Login latency.",
"id": 9,
"links": [],
"title": "IdP Requests",
"vizConfig": {
"group": "barchart",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 69,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 1,
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "short"
},
"overrides": []
},
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 0.7,
"legend": {
"calcs": [
"lastNotNull",
"min",
"mean",
"max"
],
"displayMode": "table",
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"orientation": "auto",
"showValue": "never",
"stacking": "normal",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
},
"xTickLabelRotation": 0,
"xTickLabelSpacing": 100
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-90": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"editorMode": "code",
"exemplar": false,
"expr": "sum by(resource) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[1h]))",
"instant": true,
"legendFormat": "__auto",
"range": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Account peers updates over the last hour, grouped by the resource that triggered them (policy, group, peer, route, user, network, ...). Shows which kind of configuration change generates the most fan-out work.",
"id": 90,
"links": [],
"title": "NetworkMap Triggers by source in last hour",
"vizConfig": {
"group": "piechart",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "#73BF69",
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
}
},
"overrides": []
},
"options": {
"displayLabels": [
"percent"
],
"legend": {
"displayMode": "list",
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"pieType": "donut",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"sort": "desc",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-91": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"editorMode": "builder",
"exemplar": false,
"expr": "sum by(resource) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[1h]))",
"instant": true,
"legendFormat": "__auto",
"range": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Network maps computed in the last hour, grouped by the resource whose change triggered the computation.",
"id": 91,
"links": [],
"title": "NetworkMaps by source in last hour",
"vizConfig": {
"group": "piechart",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "#73BF69",
"mode": "palette-classic"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
}
},
"overrides": []
},
"options": {
"displayLabels": [
"percent"
],
"legend": {
"displayMode": "list",
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"pieType": "donut",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"sort": "desc",
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-92": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "builder",
"exemplar": false,
"expr": "sum by(environment) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2",
"format": "time_series",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"interval": "",
"legendFormat": "{{environment}}",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Network maps computed per minute (2m increase halved). One account peers update fans out into one map per peer, so this - not the trigger count - is the real CPU driver.",
"id": 92,
"links": [],
"title": "NetworkMaps Total",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-BlYlRd",
"seriesBy": "last"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 19,
"gradientMode": "scheme",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "cpm"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
},
"panel-93": {
"kind": "Panel",
"spec": {
"data": {
"kind": "QueryGroup",
"spec": {
"queries": [
{
"kind": "PanelQuery",
"spec": {
"hidden": false,
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"disableTextWrap": false,
"editorMode": "code",
"exemplar": false,
"expr": "sum by(resource) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2",
"format": "time_series",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
"interval": "",
"legendFormat": "{{resource}}",
"range": true,
"useBackend": false
},
"version": "v0"
},
"refId": "A"
}
}
],
"queryOptions": {},
"transformations": []
}
},
"description": "Network maps computed per minute, grouped by the resource that triggered the computation.",
"id": 93,
"links": [],
"title": "NetworkMaps Total By Source",
"vizConfig": {
"group": "timeseries",
"kind": "VizConfig",
"spec": {
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic",
"seriesBy": "max"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.9,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 1,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "always",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "cpm"
},
"overrides": []
},
"options": {
"annotations": {
"clustering": -1,
"multiLane": false
},
"legend": {
"calcs": [],
"displayMode": "list",
"enableFacetedFilter": false,
"overflow": "ellipsis",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
}
},
"version": "13.2.0-30616302309"
}
}
}
},
"layout": {
"kind": "RowsLayout",
"spec": {
"rows": [
{
"kind": "RowsLayoutRow",
"spec": {
"collapse": false,
"layout": {
"kind": "GridLayout",
"spec": {
"items": [
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-14"
},
"height": 8,
"width": 12,
"x": 0,
"y": 0
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-65"
},
"height": 8,
"width": 12,
"x": 12,
"y": 0
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-15"
},
"height": 8,
"width": 12,
"x": 0,
"y": 8
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-2"
},
"height": 8,
"width": 12,
"x": 12,
"y": 8
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-23"
},
"height": 8,
"width": 12,
"x": 0,
"y": 16
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-24"
},
"height": 8,
"width": 12,
"x": 12,
"y": 16
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-16"
},
"height": 8,
"width": 12,
"x": 0,
"y": 24
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-17"
},
"height": 8,
"width": 12,
"x": 12,
"y": 24
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-84"
},
"height": 8,
"width": 12,
"x": 0,
"y": 32
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-85"
},
"height": 8,
"width": 12,
"x": 12,
"y": 32
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-86"
},
"height": 8,
"width": 12,
"x": 0,
"y": 40
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-87"
},
"height": 8,
"width": 12,
"x": 12,
"y": 40
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-26"
},
"height": 8,
"width": 12,
"x": 0,
"y": 48
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-25"
},
"height": 8,
"width": 12,
"x": 12,
"y": 48
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-34"
},
"height": 8,
"width": 12,
"x": 0,
"y": 56
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-35"
},
"height": 8,
"width": 12,
"x": 12,
"y": 56
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-31"
},
"height": 8,
"width": 12,
"x": 0,
"y": 64
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-32"
},
"height": 8,
"width": 12,
"x": 12,
"y": 64
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-9"
},
"height": 8,
"width": 12,
"x": 0,
"y": 72
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-8"
},
"height": 8,
"width": 12,
"x": 12,
"y": 72
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-29"
},
"height": 8,
"width": 12,
"x": 0,
"y": 80
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-30"
},
"height": 8,
"width": 12,
"x": 12,
"y": 80
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-66"
},
"height": 8,
"width": 12,
"x": 0,
"y": 88
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-67"
},
"height": 8,
"width": 12,
"x": 12,
"y": 88
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-10"
},
"height": 8,
"width": 12,
"x": 0,
"y": 96
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-7"
},
"height": 8,
"width": 12,
"x": 12,
"y": 96
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-33"
},
"height": 8,
"width": 12,
"x": 0,
"y": 104
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-74"
},
"height": 8,
"width": 12,
"x": 12,
"y": 104
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-11"
},
"height": 8,
"width": 12,
"x": 0,
"y": 112
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-13"
},
"height": 8,
"width": 12,
"x": 12,
"y": 112
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-12"
},
"height": 8,
"width": 12,
"x": 0,
"y": 120
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-1"
},
"height": 8,
"width": 12,
"x": 12,
"y": 120
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-27"
},
"height": 8,
"width": 12,
"x": 12,
"y": 128
}
}
]
}
},
"title": "General"
}
},
{
"kind": "RowsLayoutRow",
"spec": {
"collapse": true,
"layout": {
"kind": "GridLayout",
"spec": {
"items": [
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-76"
},
"height": 8,
"width": 24,
"x": 0,
"y": 0
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-77"
},
"height": 8,
"width": 24,
"x": 0,
"y": 8
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-78"
},
"height": 8,
"width": 24,
"x": 0,
"y": 16
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-79"
},
"height": 8,
"width": 12,
"x": 0,
"y": 24
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-80"
},
"height": 8,
"width": 12,
"x": 12,
"y": 24
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-81"
},
"height": 8,
"width": 12,
"x": 0,
"y": 32
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-82"
},
"height": 8,
"width": 12,
"x": 12,
"y": 32
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-83"
},
"height": 8,
"width": 12,
"x": 12,
"y": 40
}
}
]
}
},
"title": "NATS"
}
},
{
"kind": "RowsLayoutRow",
"spec": {
"collapse": true,
"layout": {
"kind": "GridLayout",
"spec": {
"items": [
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-46"
},
"height": 8,
"width": 12,
"x": 0,
"y": 0
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-45"
},
"height": 8,
"width": 12,
"x": 12,
"y": 0
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-47"
},
"height": 8,
"width": 12,
"x": 0,
"y": 8
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-48"
},
"height": 8,
"width": 12,
"x": 12,
"y": 8
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-50"
},
"height": 8,
"width": 12,
"x": 0,
"y": 16
}
}
]
}
},
"title": "Debugging"
}
},
{
"kind": "RowsLayoutRow",
"spec": {
"collapse": true,
"layout": {
"kind": "GridLayout",
"spec": {
"items": [
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-37"
},
"height": 8,
"width": 23,
"x": 0,
"y": 0
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-20"
},
"height": 8,
"width": 23,
"x": 0,
"y": 8
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-39"
},
"height": 8,
"width": 23,
"x": 0,
"y": 16
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-36"
},
"height": 8,
"width": 23,
"x": 0,
"y": 24
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-18"
},
"height": 8,
"width": 23,
"x": 0,
"y": 32
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-19"
},
"height": 8,
"width": 23,
"x": 0,
"y": 40
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-90"
},
"height": 19,
"width": 6,
"x": 0,
"y": 48
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-88"
},
"height": 10,
"width": 17,
"x": 6,
"y": 48
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-89"
},
"height": 9,
"width": 17,
"x": 6,
"y": 58
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-91"
},
"height": 19,
"width": 6,
"x": 0,
"y": 67
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-92"
},
"height": 10,
"width": 17,
"x": 6,
"y": 67
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-93"
},
"height": 9,
"width": 17,
"x": 6,
"y": 77
}
}
]
}
},
"title": "Network Map"
}
},
{
"kind": "RowsLayoutRow",
"spec": {
"collapse": true,
"layout": {
"kind": "GridLayout",
"spec": {
"items": [
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-52"
},
"height": 8,
"width": 12,
"x": 0,
"y": 0
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-53"
},
"height": 8,
"width": 12,
"x": 12,
"y": 0
}
},
{
"kind": "GridLayoutItem",
"spec": {
"element": {
"kind": "ElementReference",
"name": "panel-54"
},
"height": 8,
"width": 12,
"x": 0,
"y": 8
}
}
]
}
},
"title": "Browser Client"
}
}
]
}
},
"links": [],
"liveNow": false,
"preload": false,
"tags": [],
"timeSettings": {
"autoRefresh": "auto",
"autoRefreshIntervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"fiscalYearStartMonth": 0,
"from": "now-1h",
"hideTimepicker": false,
"timezone": "browser",
"to": "now"
},
"title": "Management generic",
"variables": [
{
"kind": "DatasourceVariable",
"spec": {
"allowCustomValue": true,
"current": {
"text": "",
"value": ""
},
"description": "Prometheus data source scraping the management metrics.",
"hide": "dontHide",
"includeAll": false,
"label": "Data source",
"multi": false,
"name": "datasource",
"options": [],
"pluginId": "prometheus",
"refresh": "onDashboardLoad",
"regex": "",
"skipUrlSync": false
}
},
{
"kind": "DatasourceVariable",
"spec": {
"allowCustomValue": true,
"current": {
"text": "",
"value": ""
},
"description": "Loki data source holding the management logs.",
"hide": "dontHide",
"includeAll": false,
"label": "Logs data source",
"multi": false,
"name": "logs_datasource",
"options": [],
"pluginId": "loki",
"refresh": "onDashboardLoad",
"regex": "",
"skipUrlSync": false
}
},
{
"kind": "QueryVariable",
"spec": {
"allValue": ".*",
"allowCustomValue": true,
"current": {
"text": "All",
"value": [
"$__all"
]
},
"definition": "label_values(job)",
"description": "Prometheus scrape job(s) exposing the management metrics.",
"hide": "dontHide",
"includeAll": true,
"label": "job",
"multi": true,
"name": "job",
"options": [],
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"qryType": 1,
"query": "label_values(job)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"version": "v0"
},
"refresh": "onDashboardLoad",
"regex": "",
"regexApplyTo": "value",
"skipUrlSync": false,
"sort": "alphabeticalAsc"
}
},
{
"kind": "QueryVariable",
"spec": {
"allValue": ".*",
"allowCustomValue": true,
"current": {
"text": "All",
"value": [
"$__all"
]
},
"definition": "label_values(up{job=~\"$job\"},instance)",
"description": "Instance(s) of the selected job(s). Depends on the job variable.",
"hide": "dontHide",
"includeAll": true,
"label": "instance",
"multi": true,
"name": "instance",
"options": [],
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"qryType": 1,
"query": "label_values(up{job=~\"$job\"},instance)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"version": "v0"
},
"refresh": "onDashboardLoad",
"regex": "",
"regexApplyTo": "value",
"skipUrlSync": false,
"sort": "alphabeticalAsc"
}
},
{
"kind": "QueryVariable",
"spec": {
"allowCustomValue": true,
"current": {
"text": "",
"value": ""
},
"definition": "label_values(job)",
"description": "Loki stream carrying the management logs (adjust the label if your setup does not use `job`).",
"hide": "dontHide",
"includeAll": false,
"label": "log stream",
"multi": false,
"name": "log_job",
"options": [],
"query": {
"datasource": {
"name": "${logs_datasource}"
},
"group": "loki",
"kind": "DataQuery",
"spec": {
"label": "job",
"refId": "LokiVariableQueryEditor-VariableQuery",
"type": 1
},
"version": "v0"
},
"refresh": "onDashboardLoad",
"regex": "",
"regexApplyTo": "value",
"skipUrlSync": false,
"sort": "alphabeticalAsc"
}
},
{
"kind": "CustomVariable",
"spec": {
"allValue": "5m",
"allowCustomValue": true,
"current": {
"text": "60s",
"value": "60s"
},
"hide": "dontHide",
"includeAll": false,
"label": "interval",
"multi": false,
"name": "interval",
"options": [],
"query": "5m,60s",
"skipUrlSync": false,
"valuesFormat": "csv"
}
},
{
"kind": "QueryVariable",
"spec": {
"allowCustomValue": true,
"current": {
"text": "All",
"value": "$__all"
},
"definition": "label_values(management_store_query_count_total,method)",
"hide": "hideVariable",
"includeAll": true,
"multi": true,
"name": "method",
"options": [],
"query": {
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"qryType": 1,
"query": "label_values(management_store_query_count_total,method)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"version": "v0"
},
"refresh": "onDashboardLoad",
"regex": "",
"regexApplyTo": "value",
"skipUrlSync": false,
"sort": "disabled"
}
},
{
"kind": "QueryVariable",
"spec": {
"allowCustomValue": true,
"current": {
"text": "All",
"value": [
"$__all"
]
},
"definition": "label_values(aws_rds_cpuutilization_average,dimension_DBClusterIdentifier)",
"hide": "hideVariable",
"includeAll": true,
"multi": true,
"name": "dimension_DBClusterIdentifier",
"options": [],
"query": {
"datasource": {
"name": "${datasource}"
},
"group": "prometheus",
"kind": "DataQuery",
"spec": {
"qryType": 1,
"query": "label_values(aws_rds_cpuutilization_average,dimension_DBClusterIdentifier)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"version": "v0"
},
"refresh": "onDashboardLoad",
"regex": "",
"regexApplyTo": "value",
"skipUrlSync": false,
"sort": "disabled"
}
}
]
}
Reference in New Issue View Git Blame Copy Permalink