{ "annotations": [ { "kind": "AnnotationQuery", "spec": { "builtIn": true, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "query": { "datasource": { "name": "-- Grafana --" }, "group": "grafana", "kind": "DataQuery", "spec": {}, "version": "v0" } } }, { "kind": "AnnotationQuery", "spec": { "enable": true, "hide": false, "iconColor": "red", "name": "Deployments", "query": { "datasource": { "name": "grafana" }, "group": "datasource", "kind": "DataQuery", "spec": { "expr": "", "interval": "", "refId": "Anno", "tags": [], "type": "tags" }, "version": "v0" } } } ], "cursorSync": "Crosshair", "editable": true, "elements": { "panel-1": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "histogram_quantile(0.99, sum by(le) (rate(management_grpc_updatechannel_queue_length_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", "format": "table", "fullMetaSearch": false, "includeNullMetadata": false, "instant": false, "legendFormat": "Connected grpc streams", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "p99 of the update-channel queue length: how many network map updates are waiting to be delivered to a peer. The channel buffer is 100 messages - approaching it means updates start getting dropped and peers fall behind.", "id": 1, "links": [], "title": "UpdateChannel Queue length max", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": 0 }, { "color": "green", "value": 0 }, { "color": "red", "value": 50 } ] }, "unit": "none" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.2.0-30616302309" } } }, "panel-10": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_total_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\", type=\"read\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "HTTP read duration", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "p95 duration of read API requests (GET/OPTIONS), from the total request duration histogram tagged type=read.", "id": 10, "links": [], "title": "Read HTTP API Requests p95 Duration (GET/OPTIONS)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 32, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "orange", "value": 200 } ] }, "unit": "ms" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-11": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(environment) (rate(management_updatechannel_close_one_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "CloseOne", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "expr": "sum(rate(management_updatechannel_send_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval]))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "Send", "range": true, "useBackend": false }, "version": "v0" }, "refId": "B" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum(rate(management_updatechannel_create_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60)", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "Create", "range": true, "useBackend": false }, "version": "v0" }, "refId": "C" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "rate(management_updatechannel_get_all_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "GetAll", "range": true, "useBackend": false }, "version": "v0" }, "refId": "D" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "rate(management_updatechannel_haschannel_duration_micro_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "HasChannel", "range": true, "useBackend": false }, "version": "v0" }, "refId": "E" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "rate(management_updatechannel_close_multiple_channels_count{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "CloseMultiple", "range": true, "useBackend": false }, "version": "v0" }, "refId": "F" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Rate per minute of update-channel operations: close one, send update, create, get all connected peers, has-channel and close multiple. Together they show how much channel churn an instance is handling.", "id": 11, "links": [], "title": "Update Channel operations", "vizConfig": { "group": "barchart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 69, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 1, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "fieldMinMax": false, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "reqpm" }, "overrides": [] }, "options": { "barRadius": 0, "barWidth": 0.97, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "overflow": "ellipsis", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "orientation": "auto", "showValue": "auto", "stacking": "normal", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 100 } }, "version": "13.2.0-30616302309" } } }, "panel-12": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": true, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "rate(management_updatechannel_create_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\", closed=\"true\"}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "{{host}}", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": true, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(environment) (rate(management_updatechannel_create_duration_micro_microseconds_count{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "{{host}}", "range": true, "useBackend": false }, "version": "v0" }, "refId": "B" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "__expr__" }, "group": "__expr__", "kind": "DataQuery", "spec": { "expression": "100-$A/$B*100", "type": "math" }, "version": "v0" }, "refId": "C" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Tracks how often a new peer update channel replaces a channel that was still open (closed=true), i.e. peers reconnecting before management noticed the previous stream was gone. The plotted expression is 100 - recreated/total, so the line drops as recreations rise.", "id": 12, "links": [], "title": "Percentage of Recreated channels", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "axisSoftMax": 100, "axisSoftMin": 0, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 32, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "percent" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "displayName", "value": "Recreation" } ] } ] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-13": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_close_one_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "CloseOne", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_send_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "Send", "range": true, "useBackend": false }, "version": "v0" }, "refId": "B" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_create_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "Create", "range": true, "useBackend": false }, "version": "v0" }, "refId": "C" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_get_all_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "GetAll", "range": true, "useBackend": false }, "version": "v0" }, "refId": "D" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_haschannel_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "CloseMultiple", "range": true, "useBackend": false }, "version": "v0" }, "refId": "E" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le) (rate(management_updatechannel_close_multiple_channels_bucket{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "HasChannel", "range": true, "useBackend": false }, "version": "v0" }, "refId": "F" } } ], "queryOptions": {}, "transformations": [] } }, "description": "p95 duration of each update-channel operation, in microseconds. These are in-memory operations behind a lock, so growth here means lock contention rather than I/O.", "id": 13, "links": [], "title": "Update Channel methods p95 Duration", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "fieldMinMax": false, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "µs" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-14": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "sum(management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "Connected grpc streams", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Total number of peers currently holding an open gRPC Sync stream across all management instances (management.grpc.connected.streams). A sudden drop means peers were disconnected and will come back as a Login/Sync reconnect storm.", "id": 14, "links": [], "title": "Connected peers", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": 0 }, { "color": "red", "value": 0 }, { "color": "#EAB839", "value": 8000 }, { "color": "green", "value": 11000 } ] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": true, "textMode": "auto", "wideLayout": true } }, "version": "13.2.0-30616302309" } } }, "panel-15": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "expr": "management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"}", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "{{host}}", "range": true, "useBackend": false }, "version": "v0" }, "refId": "B" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "builder", "expr": "sum by(instance) (management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"})", "instant": false, "legendFormat": "Total", "range": true }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Open peer Sync streams over time, in total per instance (A) and as raw series (B). Steps down mark instance restarts or LB rebalancing; the ramp back up is the reconnect wave.", "id": 15, "links": [], "title": "Connected peers historical", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 32, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-16": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_sync_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "GRPC Sync duration", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "p95 duration of the Sync gRPC request - the time management needs to authenticate the peer, build its first network map and open the update channel. Primary latency SLI for peer connects.", "id": 16, "links": [], "title": "gRPC Sync Request p95 Duration", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 32, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 200 } ] }, "unit": "ms" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-17": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_login_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "GRPC Login duration", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "p95 duration of the Login gRPC request (peer authentication plus initial configuration and relay credentials). Rises when the IdP, the store or account locking is slow.", "id": 17, "links": [], "title": "gRPC Login Request p95 Duration", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 32, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 200 } ] }, "unit": "ms" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-18": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(le) (increase(management_account_get_peer_network_map_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Distribution of the time taken to build the network map returned to a single peer, in milliseconds.", "id": 18, "links": [], "title": "GetPeerNetworkMap Latency", "vizConfig": { "group": "heatmap", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "calculate": false, "cellGap": 1, "color": { "exponent": 0.5, "fill": "dark-orange", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Oranges", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "placement": "bottom", "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "axisWidth": 60, "decimals": 1, "reverse": false, "unit": "ms" } } }, "version": "13.2.0-30616302309" } } }, "panel-19": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "sum by(le) (increase(management_account_network_map_object_count_objects_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Distribution of the number of objects (peers, routes, firewall rules, DNS entries and so on) in the network maps sent to clients. Large maps make both calculation and serialization more expensive.", "id": 19, "links": [], "title": "NetworkMap Objects", "vizConfig": { "group": "heatmap", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "calculate": false, "cellGap": 1, "color": { "exponent": 0.5, "fill": "dark-orange", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Oranges", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "placement": "bottom", "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "axisWidth": 60, "decimals": 0, "reverse": false, "unit": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-2": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "exemplar": false, "expr": "sum by(environment) (rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "login", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "exemplar": false, "expr": "sum by(environment) (rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "sync", "range": true, "useBackend": false }, "version": "v0" }, "refId": "B" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "exemplar": false, "expr": "sum by(environment) (rate(management_grpc_key_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "get key", "range": true, "useBackend": false }, "version": "v0" }, "refId": "C" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Rate of the three peer-facing gRPC calls: Login (authenticate, return initial config and relay credentials), Sync (open the update channel and receive network map updates) and GetServerKey (fetch the server WireGuard public key). Elevated Login/Sync rates usually mean peers are reconnecting in a loop.", "id": 2, "links": [], "title": "gRPC Requests", "vizConfig": { "group": "barchart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 51, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 1, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "options": { "barRadius": 0, "barWidth": 0.97, "fullHighlight": true, "groupWidth": 0.7, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "orientation": "auto", "showValue": "auto", "stacking": "normal", "text": {}, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 100 } }, "version": "13.2.0-30616302309" } } }, "panel-20": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(le) (increase(management_updatechannel_tosyncresponse_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Distribution of the time taken to convert a calculated network map into the gRPC SyncResponse sent to the peer, in microseconds.", "id": 20, "links": [], "title": "To Sync Response", "vizConfig": { "group": "heatmap", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "calculate": false, "cellGap": 1, "color": { "exponent": 0.5, "fill": "dark-orange", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Oranges", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "placement": "bottom", "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "axisWidth": 60, "decimals": 1, "reverse": false, "unit": "µs" } } }, "version": "13.2.0-30616302309" } } }, "panel-23": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(le) (increase(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Distribution of per-account p95 Sync durations - each sample is one account's p95, flushed every 60s. Shows whether slow Syncs are spread across the fleet or concentrated in a few accounts.", "id": 23, "links": [], "title": "GRPC Sync Latency", "vizConfig": { "group": "heatmap", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "calculate": false, "cellGap": 1, "color": { "exponent": 0.6, "fill": "dark-red", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Purples", "steps": 128 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "placement": "bottom", "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "axisWidth": 60, "decimals": 1, "reverse": false, "unit": "ms" } } }, "version": "13.2.0-30616302309" } } }, "panel-24": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(le) (increase(management_grpc_login_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Distribution of Login gRPC request durations across all peers. The right tail is what a peer experiences on startup; samples beyond 7s are what the high-latency counters in the Debugging row track.", "id": 24, "links": [], "title": "GRPC Login Latency", "vizConfig": { "group": "heatmap", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "calculate": false, "cellGap": 1, "color": { "exponent": 0.5, "fill": "dark-orange", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Purples", "steps": 128 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "placement": "bottom", "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "axisWidth": 60, "decimals": 1, "reverse": false, "unit": "ms" } } }, "version": "13.2.0-30616302309" } } }, "panel-25": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "{{host}}", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by(environment) (rate(management_grpc_login_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "instant": false, "legendFormat": "Total", "range": true }, "version": "v0" }, "refId": "B" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Rate of Login requests, per instance (A) and in total (B). Login is the most expensive peer call (IdP plus store writes); a sustained high rate means clients are restarting or failing to hold their Sync stream.", "id": 25, "links": [], "title": "gRPC Login Request rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 32, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 200 } ] }, "unit": "reqps" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-26": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "{{host}}", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "sum by(environment) (rate(management_grpc_sync_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "instant": false, "legendFormat": "Total", "range": true }, "version": "v0" }, "refId": "B" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Rate of Sync requests, per instance (A) and in total (B). Every Sync opens or re-opens a peer update channel, so a spike here is a reconnect storm and predicts CPU spent on network map calculation.", "id": 26, "links": [], "title": "gRPC Sync Request rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 32, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 200 } ] }, "unit": "reqps" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-27": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(le) (increase(management_grpc_updatechannel_queue_length_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Distribution of update-channel queue lengths across all peers. Mass moving to the right shows a growing backlog of undelivered network map updates.", "id": 27, "links": [], "title": "Update Channel heat map", "vizConfig": { "group": "heatmap", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "calculate": false, "cellGap": 1, "color": { "exponent": 0.5, "fill": "dark-orange", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Greens", "steps": 73 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "placement": "bottom", "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "axisWidth": 60, "decimals": 1, "reverse": false, "unit": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-29": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(endpoint) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval])*60) > 10", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{endpoint}} > 10", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(environment) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval])*60)", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "Total", "range": true, "useBackend": false }, "version": "v0" }, "refId": "B" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Successful (2xx) GET requests per minute broken down by endpoint, limited to endpoints above 10 rpm. Shows which read endpoints dashboard and API clients hit hardest.", "id": 29, "links": [], "title": "HTTP Read request counter", "vizConfig": { "group": "barchart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "fieldMinMax": false, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "options": { "barRadius": 0, "barWidth": 0.97, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "overflow": "ellipsis", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "orientation": "auto", "showValue": "auto", "stacking": "normal", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 100 } }, "version": "13.2.0-30616302309" } } }, "panel-30": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(endpoint) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60) > 1", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{endpoint}} > 1", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": true, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(environment) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60)", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "Total", "range": true, "useBackend": false }, "version": "v0" }, "refId": "B" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Successful (2xx) non-GET/OPTIONS requests per minute by endpoint, limited to endpoints above 1 rpm. Writes take account locks and fan out network map updates, so read this together with the Network Map row.", "id": 30, "links": [], "title": "HTTP Write request counter", "vizConfig": { "group": "barchart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "fieldMinMax": false, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "reqpm" }, "overrides": [] }, "options": { "barRadius": 0, "barWidth": 0.97, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "overflow": "ellipsis", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "orientation": "auto", "showValue": "auto", "stacking": "normal", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 100 } }, "version": "13.2.0-30616302309" } } }, "panel-31": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(le) (increase(management_store_persistence_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Distribution of how long it takes to save or delete an account in the store (management.store.persistence.duration.ms). Widening buckets point at database write pressure.", "id": 31, "links": [], "title": "Store Persistence latency", "vizConfig": { "group": "heatmap", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "calculate": false, "cellGap": 1, "color": { "exponent": 0.6, "fill": "dark-red", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Reds", "steps": 128 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "placement": "bottom", "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "axisWidth": 60, "decimals": 1, "reverse": false, "unit": "ms" } } }, "version": "13.2.0-30616302309" } } }, "panel-32": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(le) (increase(management_store_transaction_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Distribution of store transaction execution time. Nearly every API and gRPC call runs through a transaction, so this is the first place to look when latency rises everywhere at once.", "id": 32, "links": [], "title": "Store Transaction latency", "vizConfig": { "group": "heatmap", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "calculate": false, "cellGap": 1, "color": { "exponent": 0.6, "fill": "dark-red", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Reds", "steps": 128 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "placement": "bottom", "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "axisWidth": 60, "decimals": 1, "reverse": false, "unit": "ms" } } }, "version": "13.2.0-30616302309" } } }, "panel-33": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(environment) (rate(management_account_peer_meta_update_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{endpoint}} > 10", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Rate per minute of peers reporting changed metadata (OS, version, hostname, addresses). Each change persists the peer and can trigger an account peers update.", "id": 33, "links": [], "title": "Peer meta updates counter", "vizConfig": { "group": "barchart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "fieldMinMax": false, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "reqpm" }, "overrides": [] }, "options": { "barRadius": 0, "barWidth": 0.97, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "overflow": "ellipsis", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "orientation": "auto", "showValue": "auto", "stacking": "normal", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 100 } }, "version": "13.2.0-30616302309" } } }, "panel-34": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(environment) (rate(management_grpc_sync_request_blocked_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "GRPC Sync request rate", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Rate of Sync requests rejected because the peer is blocked (login expired, peer disabled or account restricted). A rising line means clients retrying without valid access.", "id": 34, "links": [], "title": "gRPC Sync Request Blocked rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 32, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 0 } ] }, "unit": "reqps" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-35": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(environment) (rate(management_grpc_login_request_blocked_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "GRPC Login request rate", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Rate of Login requests rejected because the peer is blocked. Persistent volume here is usually expired peer logins retrying in a loop.", "id": 35, "links": [], "title": "gRPC Login Request Blocked rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 32, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 0 } ] }, "unit": "reqps" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-36": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(le) (increase(management_account_update_account_peers_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Distribution of the time taken by an account peers update: preparing the data and pushing a fresh network map to every peer in the account, in milliseconds. Scales with account size.", "id": 36, "links": [], "title": "UpdateAccountPeers Latency", "vizConfig": { "group": "heatmap", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "calculate": false, "cellGap": 1, "color": { "exponent": 0.5, "fill": "dark-orange", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Oranges", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "placement": "bottom", "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "axisWidth": 60, "decimals": 1, "reverse": false, "unit": "ms" } } }, "version": "13.2.0-30616302309" } } }, "panel-37": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(le) (increase(management_updatechannel_calc_networkmap_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Distribution of the time taken to calculate one peer's network map, in milliseconds. This is the dominant CPU cost when many peers reconnect at once.", "id": 37, "links": [], "title": "Network Map Calculation", "vizConfig": { "group": "heatmap", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "calculate": false, "cellGap": 1, "color": { "exponent": 0.5, "fill": "dark-orange", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Oranges", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "placement": "bottom", "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "axisWidth": 60, "decimals": 1, "reverse": false, "unit": "ms" } } }, "version": "13.2.0-30616302309" } } }, "panel-39": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(le) (increase(management_updatechannel_calc_posturechecks_duration_micro_microseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Distribution of the time taken to evaluate a peer's posture checks while building its network map, in microseconds.", "id": 39, "links": [], "title": "Posture Check Calculation", "vizConfig": { "group": "heatmap", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "calculate": false, "cellGap": 1, "color": { "exponent": 0.5, "fill": "dark-orange", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Oranges", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "placement": "bottom", "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "axisWidth": 60, "decimals": 1, "reverse": false, "unit": "µs" } } }, "version": "13.2.0-30616302309" } } }, "panel-45": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "exemplar": false, "expr": "sort_desc(\n topk(\n 10,\n sum by (account_id) (\n increase(management_grpc_login_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n )\n )\n)", "format": "table", "instant": true, "legendFormat": "__auto", "range": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [ { "group": "organize", "kind": "Transformation", "spec": { "options": { "excludeByName": { "Time": true }, "includeByName": {}, "indexByName": {}, "renameByName": { "Time": "" } } } } ] } }, "description": "Top 10 accounts by number of Login requests that exceeded the 7s high-latency threshold.", "id": 45, "links": [], "title": "High Login Latency Ranking (login count per account)", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "footer": { "reducers": [] }, "inspect": false, "tooltip": { "placement": "auto" }, "wrapHeaderText": false }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "cellHeight": "sm", "showHeader": true, "sortBy": [ { "desc": false, "displayName": "Time" } ] } }, "version": "13.2.0-30616302309" } } }, "panel-46": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "exemplar": false, "expr": "sort_desc(\n topk(\n 10,\n sum by (account_id) (\n increase(management_grpc_sync_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n )\n )\n)", "format": "table", "instant": true, "legendFormat": "__auto", "range": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [ { "group": "organize", "kind": "Transformation", "spec": { "options": { "excludeByName": { "Time": true }, "includeByName": {}, "indexByName": {}, "renameByName": { "Time": "" } } } } ] } }, "description": "Top 10 accounts by number of Sync requests that exceeded the 7s high-latency threshold. Use it to find which tenant is behind a latency spike.", "id": 46, "links": [], "title": "High Sync Latency Ranking (sync count per account)", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "footer": { "reducers": [] }, "inspect": false, "tooltip": { "placement": "auto" }, "wrapHeaderText": false }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "cellHeight": "sm", "showHeader": true, "sortBy": [ { "desc": false, "displayName": "Time" } ] } }, "version": "13.2.0-30616302309" } } }, "panel-47": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "count(\n sum by (account_id) (\n increase(management_grpc_sync_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n ) > 1\n)", "legendFormat": "__auto", "range": true }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Number of distinct accounts with more than one slow (>7s) Sync in the interval - tells you whether a latency spike is fleet-wide or limited to a few tenants.", "id": 47, "links": [], "title": "Accounts with high Sync latency", "vizConfig": { "group": "gauge", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "barShape": "flat", "barWidthFactor": 0.5, "effects": { "barGlow": false, "centerGlow": false, "gradient": false }, "endpointMarker": "point", "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "segmentCount": 1, "segmentSpacing": 0.3, "shape": "gauge", "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto", "sparkline": false, "textMode": "auto" } }, "version": "13.2.0-30616302309" } } }, "panel-48": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "expr": "count(\n sum by (account_id) (\n increase(management_grpc_login_request_high_latency_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])\n ) > 1\n)", "legendFormat": "__auto", "range": true }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Number of distinct accounts with more than one slow (>7s) Login in the interval.", "id": 48, "links": [], "title": "Accounts with high Login latency", "vizConfig": { "group": "gauge", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "barShape": "flat", "barWidthFactor": 0.5, "effects": { "barGlow": false, "centerGlow": false, "gradient": false }, "endpointMarker": "point", "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "segmentCount": 1, "segmentSpacing": 0.3, "shape": "gauge", "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto", "sparkline": false, "textMode": "auto" } }, "version": "13.2.0-30616302309" } } }, "panel-50": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logs_datasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "direction": "backward", "editorMode": "code", "expr": "sort_desc(\ntopk(\n 20,\n sum by (peerID) (\n count_over_time(\n {job=\"$log_job\"} |= \"Sync took\" | regexp `peerID:\\s(?P[^,\\]]+)` [$__range]\n )\n )\n)\n)", "queryType": "instant" }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [ { "group": "organize", "kind": "Transformation", "spec": { "options": { "excludeByName": { "Time": true }, "includeByName": {}, "indexByName": {}, "renameByName": { "Value #A": "number of times ", "peerID": "peerPubKey" } } } } ] } }, "description": "Top 20 peers by number of Sync log lines in the selected range, parsed out of the management logs to find individual peers stuck in a reconnect loop. Pick the log stream with the `log stream` variable; the query matches the `Sync took` debug line, so management must run at debug level for this panel to have data.", "id": 50, "links": [], "title": "Highest Syncs by peer key", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "footer": { "reducers": [] }, "inspect": false, "tooltip": { "placement": "auto" }, "wrapHeaderText": false }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "labels" }, "properties": [ { "id": "custom.width", "value": 180 } ] }, { "matcher": { "id": "byName", "options": "peerPubKey" }, "properties": [ { "id": "custom.width", "value": 455 } ] } ] }, "options": { "cellHeight": "sm", "showHeader": true, "sortBy": [ { "desc": true, "displayName": "number of times " } ] } }, "version": "13.2.0-30616302309" } } }, "panel-52": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "builder", "expr": "wsproxy_active_connections{job=~\"$job\", instance=~\"$instance\"}", "legendFormat": "{{__name__}}", "range": true }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Currently open WebSocket proxy connections. The wsproxy fronts gRPC for browser-based clients, so this is the number of connected web clients.", "id": 52, "links": [], "title": "Proxy Active Connections", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-53": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "builder", "expr": "delta(wsproxy_bytes_transferred_total{job=~\"$job\", instance=~\"$instance\"}[$__interval])", "legendFormat": "{{direction}}", "range": true }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Bytes transferred through the WebSocket proxy per interval, split by direction.", "id": 53, "links": [], "title": "Proxy Traffic by Direction", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-54": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "builder", "expr": "increase(wsproxy_errors_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "legendFormat": "{{error_type}}", "range": true }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "WebSocket proxy errors grouped by error type - dial failures, read/write errors and protocol issues on browser client connections.", "id": 54, "links": [], "title": "Errors by Type", "vizConfig": { "group": "barchart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 1, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "barRadius": 0, "barWidth": 0.97, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "orientation": "auto", "showValue": "auto", "stacking": "none", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 0 } }, "version": "13.2.0-30616302309" } } }, "panel-65": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "builder", "exemplar": false, "expr": "management_grpc_connected_streams_ratio{job=~\"$job\", instance=~\"$instance\"}", "instant": true, "legendFormat": "{{host}}", "range": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Open peer Sync streams per management instance. Use it to confirm the load balancer spreads streams evenly - a skewed instance carries proportionally more update-channel and network map work.", "id": 65, "links": [], "title": "Connected Peers per Server", "vizConfig": { "group": "gauge", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "barShape": "flat", "barWidthFactor": 0.5, "effects": { "barGlow": false, "centerGlow": false, "gradient": false }, "endpointMarker": "point", "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "segmentCount": 1, "segmentSpacing": 0.3, "shape": "gauge", "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto", "sparkline": false, "textMode": "auto" } }, "version": "13.2.0-30616302309" } } }, "panel-66": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(host) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method=\"GET\", code=~\"^2.+\"}[$__rate_interval]) * 60)", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{host}}", "range": true, "useBackend": false }, "version": "v0" }, "refId": "B" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Successful GET requests per minute grouped by management instance - read load distribution across the fleet.", "id": 66, "links": [], "title": "HTTP Read request counter", "vizConfig": { "group": "barchart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "fieldMinMax": false, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "options": { "barRadius": 0, "barWidth": 0.97, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "overflow": "ellipsis", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "orientation": "auto", "showValue": "auto", "stacking": "normal", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 100 } }, "version": "13.2.0-30616302309" } } }, "panel-67": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(host) (rate(management_http_response_counter_total{job=~\"$job\", instance=~\"$instance\", method!~\"(GET|OPTIONS)\", code=~\"^2.+\"}[$__rate_interval])*60)", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{host}}", "range": true, "useBackend": false }, "version": "v0" }, "refId": "B" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Successful write requests per minute grouped by management instance - write load distribution across the fleet.", "id": 67, "links": [], "title": "HTTP Write request counter", "vizConfig": { "group": "barchart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 0, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "fieldMinMax": false, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "reqpm" }, "overrides": [] }, "options": { "barRadius": 0, "barWidth": 0.97, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "overflow": "ellipsis", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "orientation": "auto", "showValue": "auto", "stacking": "normal", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 100 } }, "version": "13.2.0-30616302309" } } }, "panel-7": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_total_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\", type=\"write\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "HTTP write duration", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "p95 duration of write API requests (PUT/POST/DELETE), tagged type=write. Writes acquire account locks and trigger peer updates, so they are normally slower than reads.", "id": 7, "links": [], "title": "Write HTTP API Requests p95 Duration (PUT/POST/DELETE)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 32, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 200 } ] }, "unit": "ms" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-74": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(le) (increase(management_pat_usage_distribution_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Distribution of personal access token usage counts, sampled once per minute per token. Mass in the high buckets means a single PAT is hammering the API.", "id": 74, "links": [], "title": "PAT usage counter", "vizConfig": { "group": "heatmap", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "calculate": false, "cellGap": 1, "color": { "exponent": 0.5, "fill": "dark-orange", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Purples", "steps": 128 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "placement": "bottom", "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "axisWidth": 60, "decimals": 1, "reverse": false, "unit": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-76": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "exemplar": false, "expr": "gnatsd_connz_subscriptions{job=~\"$job\", instance=~\"$instance\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, "legendFormat": "{{host}}", "range": false, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Current subscriptions held by each client connected to NATS (connz). Signal instances subscribe per connected peer, so this tracks how peers are spread across signal nodes.", "id": 76, "links": [], "title": "Subscriptions per node", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.2.0-30616302309" } } }, "panel-77": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "exemplar": false, "expr": "gnatsd_connz_subscriptions{job=~\"$job\", instance=~\"$instance\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "{{host}}", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Subscriptions per connected NATS client over time. Steps mark signal instances restarting and re-subscribing their peers.", "id": 77, "links": [], "title": "Subscriptions per node", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-78": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "expr": "gnatsd_connz_num_connections{job=~\"$job\", instance=~\"$instance\"}", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{host}}", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Number of client connections on each NATS node - effectively how many signal instances are attached to that node.", "id": 78, "links": [], "title": "Signal nodes connected per NATs", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.2.0-30616302309" } } }, "panel-79": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(environment) (rate(gnatsd_connz_out_msgs{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Rate of messages NATS delivers out to its clients: the peer signalling throughput flowing through the cluster.", "id": 79, "links": [], "title": "Messages per Second", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "mps" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-8": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le) (rate(management_http_request_duration_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "HTTP request duration", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "p95 duration of incoming REST API requests by endpoint and method (management.http.request.duration.ms). Covers dashboard and API clients only - peer traffic is on the gRPC panels.", "id": 8, "links": [], "title": "HTTP API Requests p95 Duration", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 32, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 200 } ] }, "unit": "ms" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-80": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(environment) (rate(gnatsd_connz_out_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Rate of bytes NATS delivers out to its clients - signalling bandwidth, useful next to the message rate to spot unusually large payloads.", "id": 80, "links": [], "title": "Data per Second", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-81": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "expr": "sum by(environment) (increase(gnatsd_varz_slow_consumers{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Increase of the NATS slow-consumer counter. Anything above zero means a signal node could not read fast enough and NATS dropped its connection, so peers lose signalling until it reconnects.", "id": 81, "links": [], "title": "Slow consumers detected", "vizConfig": { "group": "barchart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 80, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 1, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "options": { "barRadius": 0, "barWidth": 0.97, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "orientation": "auto", "showValue": "auto", "stacking": "none", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 100 } }, "version": "13.2.0-30616302309" } } }, "panel-82": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "expr": "gnatsd_varz_subscriptions{job=~\"$job\", instance=~\"$instance\"}", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{host}}", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Total subscriptions registered on each NATS server (varz). Grows roughly with the number of connected peers.", "id": 82, "links": [], "title": "Total number of NATS subscriptions", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-83": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{instance}}", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Resident memory of the scraped processes. process_resident_memory_bytes is exposed by every Go/Prometheus process, so narrow the job and instance variables to the NATS nodes to read this panel as NATS memory.", "id": 83, "links": [], "title": "NATS memory consumption", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-84": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "idelta(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[2m])", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Per-account Sync p95 samples as a heatmap (2m deltas). Use it to spot a handful of large or slow accounts pulling the overall Sync latency up.", "id": 84, "links": [], "title": "GRPC Sync Latency By Account", "vizConfig": { "group": "heatmap", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "calculate": false, "cellGap": 1, "color": { "exponent": 0.6, "fill": "dark-red", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Purples", "steps": 128 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "placement": "bottom", "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "axisWidth": 60, "decimals": 1, "reverse": false, "unit": "ms" } } }, "version": "13.2.0-30616302309" } } }, "panel-85": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "idelta(management_grpc_login_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[2m])", "format": "heatmap", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Per-account Login p95 samples as a heatmap (2m deltas). Highlights accounts whose logins are much slower than the fleet average.", "id": 85, "links": [], "title": "GRPC Login Latency By Account", "vizConfig": { "group": "heatmap", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } } }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "calculate": false, "cellGap": 1, "color": { "exponent": 0.5, "fill": "dark-orange", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "Purples", "steps": 128 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1e-09 }, "legend": { "placement": "bottom", "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "mode": "single", "showColorScale": false, "yHistogram": false }, "yAxis": { "axisPlacement": "left", "axisWidth": 60, "decimals": 1, "reverse": false, "unit": "ms" } } }, "version": "13.2.0-30616302309" } } }, "panel-86": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_sync_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "GRPC Sync duration", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "p95 of the per-account Sync p95 distribution - the latency seen by the worst-served accounts rather than by the average request. Diverges from the global p95 when only a few accounts are hurting.", "id": 86, "links": [], "title": "gRPC Sync Request p95 Duration (by Account)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 32, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 500 } ] }, "unit": "ms" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-87": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le) (rate(management_grpc_login_request_duration_p95_by_account_ms_milliseconds_bucket{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "GRPC Login duration", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "p95 of the per-account Login p95 distribution - login latency for the worst-served accounts rather than the average request.", "id": 87, "links": [], "title": "gRPC Login Request p95 Duration (by Account)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 32, "gradientMode": "hue", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 200 } ] }, "unit": "ms" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-88": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "exemplar": false, "expr": "sum by(environment) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "legendFormat": "{{environment}}", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Account peers updates triggered per minute (2m increase halved). Each trigger recalculates and pushes network maps to every peer in the account.", "id": 88, "links": [], "title": "NetworkMap Triggers Total", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "continuous-BlYlRd", "seriesBy": "last" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 19, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineStyle": { "fill": "solid" }, "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "cpm" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-89": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "sum by(resource) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m]))", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "legendFormat": "__auto", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Account peers updates over 2m intervals, grouped by the resource that triggered them - the time series behind the pie chart above.", "id": 89, "links": [], "title": "NetworkMap Triggers By Source", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic", "seriesBy": "max" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.9, "drawStyle": "line", "fillOpacity": 50, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 2, "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "cpm" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-9": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "sum by(environment) (rate(management_idp_authenticate_request_counter_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])*60)", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "authenticate (mgtm->auth0)", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "rate(management_idp_update_user_meta_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$interval])*60", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "update user meta", "range": true, "useBackend": false }, "version": "v0" }, "refId": "B" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "expr": "rate(management_idp_get_account_counter_total{job=~\"$job\", instance=~\"$instance\", host=\"$host\"}[$__rate_interval])*60", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "get account", "range": true, "useBackend": false }, "version": "v0" }, "refId": "C" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Requests per minute that management sends to the configured identity provider: authenticate (service token refresh), user metadata updates and account lookups. IdP throttling shows up here before it shows up as Login latency.", "id": 9, "links": [], "title": "IdP Requests", "vizConfig": { "group": "barchart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "fillOpacity": 69, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 1, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 } ] }, "unit": "short" }, "overrides": [] }, "options": { "barRadius": 0, "barWidth": 0.97, "fullHighlight": false, "groupWidth": 0.7, "legend": { "calcs": [ "lastNotNull", "min", "mean", "max" ], "displayMode": "table", "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "orientation": "auto", "showValue": "never", "stacking": "normal", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, "xTickLabelRotation": 0, "xTickLabelSpacing": 100 } }, "version": "13.2.0-30616302309" } } }, "panel-90": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "code", "exemplar": false, "expr": "sum by(resource) (increase(management_account_update_account_peers_counter_total{job=~\"$job\", instance=~\"$instance\"}[1h]))", "instant": true, "legendFormat": "__auto", "range": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Account peers updates over the last hour, grouped by the resource that triggered them (policy, group, peer, route, user, network, ...). Shows which kind of configuration change generates the most fan-out work.", "id": 90, "links": [], "title": "NetworkMap Triggers by source in last hour", "vizConfig": { "group": "piechart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#73BF69", "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } } }, "overrides": [] }, "options": { "displayLabels": [ "percent" ], "legend": { "displayMode": "list", "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "pieType": "donut", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "sort": "desc", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-91": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "editorMode": "builder", "exemplar": false, "expr": "sum by(resource) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[1h]))", "instant": true, "legendFormat": "__auto", "range": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Network maps computed in the last hour, grouped by the resource whose change triggered the computation.", "id": 91, "links": [], "title": "NetworkMaps by source in last hour", "vizConfig": { "group": "piechart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#73BF69", "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } } }, "overrides": [] }, "options": { "displayLabels": [ "percent" ], "legend": { "displayMode": "list", "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "pieType": "donut", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "sort": "desc", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-92": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "builder", "exemplar": false, "expr": "sum by(environment) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "legendFormat": "{{environment}}", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Network maps computed per minute (2m increase halved). One account peers update fans out into one map per peer, so this - not the trigger count - is the real CPU driver.", "id": 92, "links": [], "title": "NetworkMaps Total", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "continuous-BlYlRd", "seriesBy": "last" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 19, "gradientMode": "scheme", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineStyle": { "fill": "solid" }, "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "cpm" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } }, "panel-93": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "sum by(resource) (increase(management_network_map_counter_total{job=~\"$job\", instance=~\"$instance\"}[2m])) / 2", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "interval": "", "legendFormat": "{{resource}}", "range": true, "useBackend": false }, "version": "v0" }, "refId": "A" } } ], "queryOptions": {}, "transformations": [] } }, "description": "Network maps computed per minute, grouped by the resource that triggered the computation.", "id": 93, "links": [], "title": "NetworkMaps Total By Source", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic", "seriesBy": "max" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.9, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 1, "scaleDistribution": { "type": "linear" }, "showPoints": "always", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "cpm" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "enableFacetedFilter": false, "overflow": "ellipsis", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.2.0-30616302309" } } } }, "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-14" }, "height": 8, "width": 12, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-65" }, "height": 8, "width": 12, "x": 12, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-15" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-2" }, "height": 8, "width": 12, "x": 12, "y": 8 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-23" }, "height": 8, "width": 12, "x": 0, "y": 16 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-24" }, "height": 8, "width": 12, "x": 12, "y": 16 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-16" }, "height": 8, "width": 12, "x": 0, "y": 24 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-17" }, "height": 8, "width": 12, "x": 12, "y": 24 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-84" }, "height": 8, "width": 12, "x": 0, "y": 32 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-85" }, "height": 8, "width": 12, "x": 12, "y": 32 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-86" }, "height": 8, "width": 12, "x": 0, "y": 40 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-87" }, "height": 8, "width": 12, "x": 12, "y": 40 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-26" }, "height": 8, "width": 12, "x": 0, "y": 48 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-25" }, "height": 8, "width": 12, "x": 12, "y": 48 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-34" }, "height": 8, "width": 12, "x": 0, "y": 56 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-35" }, "height": 8, "width": 12, "x": 12, "y": 56 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-31" }, "height": 8, "width": 12, "x": 0, "y": 64 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-32" }, "height": 8, "width": 12, "x": 12, "y": 64 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-9" }, "height": 8, "width": 12, "x": 0, "y": 72 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-8" }, "height": 8, "width": 12, "x": 12, "y": 72 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-29" }, "height": 8, "width": 12, "x": 0, "y": 80 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-30" }, "height": 8, "width": 12, "x": 12, "y": 80 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-66" }, "height": 8, "width": 12, "x": 0, "y": 88 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-67" }, "height": 8, "width": 12, "x": 12, "y": 88 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-10" }, "height": 8, "width": 12, "x": 0, "y": 96 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-7" }, "height": 8, "width": 12, "x": 12, "y": 96 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-33" }, "height": 8, "width": 12, "x": 0, "y": 104 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-74" }, "height": 8, "width": 12, "x": 12, "y": 104 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-11" }, "height": 8, "width": 12, "x": 0, "y": 112 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-13" }, "height": 8, "width": 12, "x": 12, "y": 112 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-12" }, "height": 8, "width": 12, "x": 0, "y": 120 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-1" }, "height": 8, "width": 12, "x": 12, "y": 120 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-27" }, "height": 8, "width": 12, "x": 12, "y": 128 } } ] } }, "title": "General" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": true, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-76" }, "height": 8, "width": 24, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-77" }, "height": 8, "width": 24, "x": 0, "y": 8 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-78" }, "height": 8, "width": 24, "x": 0, "y": 16 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-79" }, "height": 8, "width": 12, "x": 0, "y": 24 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-80" }, "height": 8, "width": 12, "x": 12, "y": 24 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-81" }, "height": 8, "width": 12, "x": 0, "y": 32 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-82" }, "height": 8, "width": 12, "x": 12, "y": 32 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-83" }, "height": 8, "width": 12, "x": 12, "y": 40 } } ] } }, "title": "NATS" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": true, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-46" }, "height": 8, "width": 12, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-45" }, "height": 8, "width": 12, "x": 12, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-47" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-48" }, "height": 8, "width": 12, "x": 12, "y": 8 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-50" }, "height": 8, "width": 12, "x": 0, "y": 16 } } ] } }, "title": "Debugging" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": true, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-37" }, "height": 8, "width": 23, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-20" }, "height": 8, "width": 23, "x": 0, "y": 8 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-39" }, "height": 8, "width": 23, "x": 0, "y": 16 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-36" }, "height": 8, "width": 23, "x": 0, "y": 24 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-18" }, "height": 8, "width": 23, "x": 0, "y": 32 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-19" }, "height": 8, "width": 23, "x": 0, "y": 40 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-90" }, "height": 19, "width": 6, "x": 0, "y": 48 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-88" }, "height": 10, "width": 17, "x": 6, "y": 48 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-89" }, "height": 9, "width": 17, "x": 6, "y": 58 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-91" }, "height": 19, "width": 6, "x": 0, "y": 67 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-92" }, "height": 10, "width": 17, "x": 6, "y": 67 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-93" }, "height": 9, "width": 17, "x": 6, "y": 77 } } ] } }, "title": "Network Map" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": true, "layout": { "kind": "GridLayout", "spec": { "items": [ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-52" }, "height": 8, "width": 12, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-53" }, "height": 8, "width": 12, "x": 12, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-54" }, "height": 8, "width": 12, "x": 0, "y": 8 } } ] } }, "title": "Browser Client" } } ] } }, "links": [], "liveNow": false, "preload": false, "tags": [], "timeSettings": { "autoRefresh": "auto", "autoRefreshIntervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "fiscalYearStartMonth": 0, "from": "now-1h", "hideTimepicker": false, "timezone": "browser", "to": "now" }, "title": "Management generic", "variables": [ { "kind": "DatasourceVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "description": "Prometheus data source scraping the management metrics.", "hide": "dontHide", "includeAll": false, "label": "Data source", "multi": false, "name": "datasource", "options": [], "pluginId": "prometheus", "refresh": "onDashboardLoad", "regex": "", "skipUrlSync": false } }, { "kind": "DatasourceVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "description": "Loki data source holding the management logs.", "hide": "dontHide", "includeAll": false, "label": "Logs data source", "multi": false, "name": "logs_datasource", "options": [], "pluginId": "loki", "refresh": "onDashboardLoad", "regex": "", "skipUrlSync": false } }, { "kind": "QueryVariable", "spec": { "allValue": ".*", "allowCustomValue": true, "current": { "text": "All", "value": [ "$__all" ] }, "definition": "label_values(job)", "description": "Prometheus scrape job(s) exposing the management metrics.", "hide": "dontHide", "includeAll": true, "label": "job", "multi": true, "name": "job", "options": [], "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "qryType": 1, "query": "label_values(job)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "version": "v0" }, "refresh": "onDashboardLoad", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } }, { "kind": "QueryVariable", "spec": { "allValue": ".*", "allowCustomValue": true, "current": { "text": "All", "value": [ "$__all" ] }, "definition": "label_values(up{job=~\"$job\"},instance)", "description": "Instance(s) of the selected job(s). Depends on the job variable.", "hide": "dontHide", "includeAll": true, "label": "instance", "multi": true, "name": "instance", "options": [], "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "qryType": 1, "query": "label_values(up{job=~\"$job\"},instance)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "version": "v0" }, "refresh": "onDashboardLoad", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(job)", "description": "Loki stream carrying the management logs (adjust the label if your setup does not use `job`).", "hide": "dontHide", "includeAll": false, "label": "log stream", "multi": false, "name": "log_job", "options": [], "query": { "datasource": { "name": "${logs_datasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "label": "job", "refId": "LokiVariableQueryEditor-VariableQuery", "type": 1 }, "version": "v0" }, "refresh": "onDashboardLoad", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } }, { "kind": "CustomVariable", "spec": { "allValue": "5m", "allowCustomValue": true, "current": { "text": "60s", "value": "60s" }, "hide": "dontHide", "includeAll": false, "label": "interval", "multi": false, "name": "interval", "options": [], "query": "5m,60s", "skipUrlSync": false, "valuesFormat": "csv" } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "All", "value": "$__all" }, "definition": "label_values(management_store_query_count_total,method)", "hide": "hideVariable", "includeAll": true, "multi": true, "name": "method", "options": [], "query": { "group": "prometheus", "kind": "DataQuery", "spec": { "qryType": 1, "query": "label_values(management_store_query_count_total,method)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "version": "v0" }, "refresh": "onDashboardLoad", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "disabled" } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "All", "value": [ "$__all" ] }, "definition": "label_values(aws_rds_cpuutilization_average,dimension_DBClusterIdentifier)", "hide": "hideVariable", "includeAll": true, "multi": true, "name": "dimension_DBClusterIdentifier", "options": [], "query": { "datasource": { "name": "${datasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "qryType": 1, "query": "label_values(aws_rds_cpuutilization_average,dimension_DBClusterIdentifier)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "version": "v0" }, "refresh": "onDashboardLoad", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "disabled" } } ] }