diff --git a/infrastructure_files/observability/grafana/dashboards/envoy.json b/infrastructure_files/observability/grafana/dashboards/envoy.json new file mode 100644 index 000000000..fba172f65 --- /dev/null +++ b/infrastructure_files/observability/grafana/dashboards/envoy.json @@ -0,0 +1,4002 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "Saturation and backpressure", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Open client connections per Envoy instance, and the rate of new connections. A flat active line with a rising accept rate means connections are being closed as fast as they arrive.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(envoy_http_downstream_cx_active{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} active", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_http_downstream_cx_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} new/s", + "range": true, + "refId": "B" + } + ], + "title": "Downstream connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Connections Envoy refused before any routing happened. Non-zero cx_overflow means a listener or global connection limit is capping the load balancer, not the backends.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_listener_downstream_cx_overflow{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} listener overflow", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_listener_downstream_global_cx_overflow{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} global overflow", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_listener_no_filter_chain_match{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} no filter chain match", + "range": true, + "refId": "C" + } + ], + "title": "Connection rejections (listener)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Envoy's own self-protection. Heap pressure approaching 1 or any active overload action means Envoy is already shedding load: it will stop accepting requests/connections before it OOMs.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "envoy_overload_envoy_resource_monitors_fixed_heap_pressure{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} heap pressure", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "envoy_overload_envoy_resource_monitors_global_downstream_max_connections_pressure{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} downstream cx pressure", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "envoy_overload_envoy_overload_actions_stop_accepting_requests_active{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} stop accepting requests", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "envoy_overload_envoy_overload_actions_stop_accepting_connections_active{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} stop accepting connections", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "envoy_overload_envoy_overload_actions_shrink_heap_active{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} shrink heap", + "range": true, + "refId": "E" + } + ], + "title": "Overload manager", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Requests waiting for a free upstream connection. This is the clearest early sign that backends are the bottleneck: it rises well before latency or errors do.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(envoy_cluster_upstream_rq_pending_active{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream queue depth (pending requests)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "1 means that circuit breaker is tripped and Envoy is rejecting work for that upstream cluster right now. Which one opens tells you the limit to raise: cx, rq_pending, rq or rq_retry.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 7 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(envoy_cluster_circuit_breakers_default_cx_open{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} cx", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(envoy_cluster_circuit_breakers_default_rq_pending_open{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} rq_pending", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(envoy_cluster_circuit_breakers_default_rq_open{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} rq", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(envoy_cluster_circuit_breakers_default_rq_retry_open{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} rq_retry", + "range": true, + "refId": "D" + } + ], + "title": "Circuit breakers open", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How much of each budget is left before the breaker trips. A value trending to 0 is the warning that the panel to its left is about to go to 1.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(envoy_cluster_circuit_breakers_default_remaining_cx{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} remaining cx", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(envoy_cluster_circuit_breakers_default_remaining_pending{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} remaining pending", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(envoy_cluster_circuit_breakers_default_remaining_rq{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} remaining rq", + "range": true, + "refId": "C" + } + ], + "title": "Circuit breaker headroom", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Work Envoy dropped itself rather than passing upstream. Any of these is user-visible failure caused by a limit in Envoy's own configuration.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 13 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_pending_overflow{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} pending overflow", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_cx_pool_overflow{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} cx pool overflow", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_retry_overflow{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} retry overflow", + "range": true, + "refId": "C" + } + ], + "title": "Load shed by Envoy", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Concurrent requests being served downstream, and requests started per second. Divergence between the two means requests are taking longer to complete.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 13 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(envoy_http_downstream_rq_active{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} active", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_http_downstream_rq_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} rq/s", + "range": true, + "refId": "B" + } + ], + "title": "Requests in flight", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Open upstream connections divided by healthy hosts. A sharp climb here while request rate is flat means connections are piling up because backends are answering more slowly.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 13 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(envoy_cluster_upstream_cx_active{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name) / clamp_min(sum(envoy_cluster_membership_healthy{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name), 1)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream connections per healthy host", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 12, + "panels": [], + "title": "Latency", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "End-to-end latency as the client sees it, including time queued inside Envoy.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50,sum(rate(envoy_http_downstream_rq_time_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (le))", + "format": "time_series", + "instant": false, + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90,sum(rate(envoy_http_downstream_rq_time_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (le))", + "format": "time_series", + "instant": false, + "legendFormat": "p90", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99,sum(rate(envoy_http_downstream_rq_time_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (le))", + "format": "time_series", + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "Downstream request duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Backend-side latency per upstream cluster. Compare with the panel to the left to see whether slowness originates upstream or inside Envoy.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99,sum(rate(envoy_cluster_upstream_rq_time_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (le,envoy_cluster_name))", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} p99", + "range": true, + "refId": "A" + } + ], + "title": "Upstream request duration p99 by cluster", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Downstream p99 minus upstream p99: the latency Envoy itself adds, mostly connection-pool queueing. If this grows while upstream p99 is flat, the load balancer is the bottleneck.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99,sum(rate(envoy_http_downstream_rq_time_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (le)) - histogram_quantile(0.99,sum(rate(envoy_cluster_upstream_rq_time_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (le))", + "format": "time_series", + "instant": false, + "legendFormat": "p99 overhead", + "range": true, + "refId": "A" + } + ], + "title": "Time spent inside Envoy (p99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "TCP/TLS connect time to backends. A spike here with healthy request latency points at the network, conntrack tables or backend accept queues rather than application code.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 26 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99,sum(rate(envoy_cluster_upstream_cx_connect_ms_bucket{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (le,envoy_cluster_name))", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} p99", + "range": true, + "refId": "A" + } + ], + "title": "Upstream connect time p99", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Bytes moved downstream (received positive, sent negative) per Envoy instance. Useful for spotting a single instance carrying a disproportionate share of traffic.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 26 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_http_downstream_cx_rx_bytes_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} rx", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_http_downstream_cx_tx_bytes_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance) * -1", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} tx", + "range": true, + "refId": "B" + } + ], + "title": "Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Traffic split across load balancer instances. A persistent imbalance means the layer in front of Envoy (DNS, cloud LB, keepalive reuse) is not spreading load evenly.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 26 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_http_downstream_rq_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Request rate per Envoy instance", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 19, + "panels": [], + "title": "Errors and retries", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Response codes returned to clients, stacked. Envoy's own rejections (503 on circuit breaker, 429 on rate limit) show up here as 5xx/4xx even when backends are fine.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 33 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_http_downstream_rq_xx{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (envoy_response_code_class)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_response_code_class}}xx", + "range": true, + "refId": "A" + } + ], + "title": "Downstream responses by class", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Server errors as a share of all responses. Threshold lines at 1% and 5%.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 33 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "100 * sum(rate(envoy_http_downstream_rq_xx{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_response_code_class=\"5\"}[$__rate_interval])) / clamp_min(sum(rate(envoy_http_downstream_rq_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])), 1)", + "format": "time_series", + "instant": false, + "legendFormat": "5xx %", + "range": true, + "refId": "A" + } + ], + "title": "Downstream 5xx ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Which backend cluster is producing the errors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 33 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_xx{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\",envoy_response_code_class=\"5\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Upstream 5xx by cluster", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Failures reaching or completing against backends. connect_timeout and rq_timeout separate 'cannot open a connection' from 'connection opened but the response never came'.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_cx_connect_fail{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} connect fail", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_cx_connect_timeout{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} connect timeout", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_timeout{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} rq timeout", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_cx_destroy_remote_with_active_rq{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} closed with active rq", + "range": true, + "refId": "D" + } + ], + "title": "Upstream connection and request failures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Retries amplify load on an already-struggling backend. retry_overflow means the retry budget is exhausted and requests are failing without a second attempt.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 39 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_retry{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} retries", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_retry_success{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} retry success", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_retry_overflow{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} retry overflow", + "range": true, + "refId": "C" + } + ], + "title": "Retries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Stream resets and handshake failures. A rise in downstream protocol errors or TLS errors under load often means clients are timing out and abandoning connections.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 39 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_rx_reset{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} rx reset", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_rq_tx_reset{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} tx reset", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_http_downstream_cx_protocol_error{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} downstream protocol error", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_listener_ssl_connection_error{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} TLS error", + "range": true, + "refId": "D" + } + ], + "title": "Resets, protocol and TLS errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 26, + "panels": [], + "title": "Upstream health and load balancing", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Healthy, degraded and total hosts per upstream cluster. Losing hosts concentrates the same traffic on fewer backends, which is a common cause of sudden pressure.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 46 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(envoy_cluster_membership_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} total", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(envoy_cluster_membership_healthy{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} healthy", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(envoy_cluster_membership_degraded{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} degraded", + "range": true, + "refId": "C" + } + ], + "title": "Cluster membership", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Share of each cluster that is currently healthy. Below the panic threshold (50% by default) Envoy stops excluding unhealthy hosts entirely.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "green", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 46 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "100 * sum(envoy_cluster_membership_healthy{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name) / clamp_min(sum(envoy_cluster_membership_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name), 1)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Healthy host ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Non-zero means too few healthy hosts remain, so Envoy is load balancing across all hosts including unhealthy ones. Requests will fail while this is firing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.001 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 46 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_lb_healthy_panic{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}}", + "range": true, + "refId": "A" + } + ], + "title": "LB panic mode", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Hosts Envoy pulled out of rotation for failing. Ejections shrink the pool and can cascade: each ejection raises load on the hosts that remain.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 52 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(envoy_cluster_outlier_detection_ejections_active{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} active", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_outlier_detection_ejections_enforced_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} enforced/s", + "range": true, + "refId": "B" + } + ], + "title": "Outlier detection ejections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Health check failures and timeouts per cluster, before ejection kicks in.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 52 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_health_check_failure{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} failure", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_health_check_network_failure{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} network failure", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_health_check_verify_cluster{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} verify cluster", + "range": true, + "refId": "C" + } + ], + "title": "Active health checks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "New upstream connections per second against destroys. High churn means connections are not being reused, so every request pays connect and TLS cost.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 52 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_cx_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} new/s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_upstream_cx_destroy{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\",envoy_cluster_name=~\"$upstream\"}[$__rate_interval])) by (envoy_cluster_name)", + "format": "time_series", + "instant": false, + "legendFormat": "{{envoy_cluster_name}} destroyed/s", + "range": true, + "refId": "B" + } + ], + "title": "Upstream connection churn", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 33, + "panels": [], + "title": "Envoy process health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Allocated vs heap vs physical memory. Allocated tracking heap upward while traffic is flat usually means buffered data from slow backends or slow clients.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 59 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "envoy_server_memory_allocated{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} allocated", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "envoy_server_memory_heap_size{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} heap", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "envoy_server_memory_physical_size{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} physical", + "range": true, + "refId": "C" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Worker threads that failed to check in on time. This is the strongest signal that Envoy is CPU starved: the event loop is blocked and every request on that worker stalls.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.001 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 59 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_server_watchdog_miss{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} miss", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_server_watchdog_mega_miss{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} mega miss", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_server_main_thread_watchdog_miss{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} main thread miss", + "range": true, + "refId": "C" + } + ], + "title": "Watchdog misses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "envoy_server_state is 0 when live; anything else means draining, initializing or pre-initializing. total_connections is the process-wide count the global connection limit applies to.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 59 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "envoy_server_state{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} state", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "envoy_server_total_connections{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} total connections", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "envoy_server_parent_connections{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} parent connections", + "range": true, + "refId": "C" + } + ], + "title": "Server state and connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "A resetting uptime means the process restarted; a rising hot restart epoch means a config reload swapped the binary. Both explain sudden metric discontinuities elsewhere on this dashboard.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 65 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "envoy_server_uptime{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} uptime", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "envoy_server_hot_restart_epoch{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} hot restart epoch", + "range": true, + "refId": "B" + } + ], + "title": "Uptime and hot restarts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "connected_state 0 means Envoy has lost its control plane and is serving the last config it received. Update failures mean new config is being rejected.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 65 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "envoy_control_plane_connected_state{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} connected", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_manager_cds_update_failure{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} CDS failure", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_cluster_manager_cds_update_rejected{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} CDS rejected", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_listener_manager_lds_update_failure{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} LDS failure", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(envoy_listener_manager_lds_update_rejected{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"}[$__rate_interval])) by (instance)", + "format": "time_series", + "instant": false, + "legendFormat": "{{instance}} LDS rejected", + "range": true, + "refId": "E" + } + ], + "title": "xDS control plane", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Minimum days to expiry across all configured TLS certificates.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 14 + }, + { + "color": "green", + "value": 30 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 65 + }, + "id": 39, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(envoy_server_days_until_first_cert_expiring{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\",instance=~\"$host\"})", + "format": "time_series", + "instant": false, + "legendFormat": "days", + "range": true, + "refId": "A" + } + ], + "title": "Days until first certificate expires", + "type": "stat" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "envoy", + "load-balancer" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "Prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(envoy_server_live,cluster)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(envoy_server_live,cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(envoy_server_live{cluster=~\"$cluster\"},environment)", + "hide": 0, + "includeAll": false, + "label": "environment", + "multi": false, + "name": "environment", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(envoy_server_live{cluster=~\"$cluster\"},environment)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "envoy", + "value": "envoy" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(envoy_server_live{cluster=~\"$cluster\",environment=~\"$environment\"},job)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(envoy_server_live{cluster=~\"$cluster\",environment=~\"$environment\"},job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(envoy_server_live{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\"},instance)", + "hide": 0, + "includeAll": true, + "label": "host", + "multi": true, + "name": "host", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(envoy_server_live{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(envoy_cluster_upstream_rq_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\"},envoy_cluster_name)", + "hide": 0, + "includeAll": true, + "label": "upstream cluster", + "multi": true, + "name": "upstream", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(envoy_cluster_upstream_rq_total{cluster=~\"$cluster\",environment=~\"$environment\",job=~\"$job\"},envoy_cluster_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Netbird / Envoy load balancers", + "uid": "netbird-envoy-lb", + "version": 1, + "weekStart": "" +} diff --git a/infrastructure_files/observability/grafana/dashboards/readme.md b/infrastructure_files/observability/grafana/dashboards/readme.md index 3160921a6..fae6995e5 100644 --- a/infrastructure_files/observability/grafana/dashboards/readme.md +++ b/infrastructure_files/observability/grafana/dashboards/readme.md @@ -9,4 +9,44 @@ NOTE: - Your installation may have a subset of these variables. -- The dashboard expects `exported_endpoint` instead of `endpoint` in HTTP request metrics. \ No newline at end of file +- The dashboard expects `exported_endpoint` instead of `endpoint` in HTTP request metrics. +## Envoy load balancer dashboard (`envoy.json`) + +Covers the Envoy instances in front of NetBird. It is built for one question: +*when the load balancers are under pressure, where is the pressure coming from?* + +Read the rows in order — each one rules out a layer: + +1. **Saturation and backpressure** — is Envoy itself the limit? Connection + rejections, the overload manager, circuit breakers and the upstream pending + queue all show Envoy shedding load before the backends are even involved. + Queue depth and circuit breaker headroom move first, well before latency does. +2. **Latency** — *Time spent inside Envoy* is downstream p99 minus upstream p99. + If it grows while upstream p99 stays flat, the load balancer is the bottleneck; + if both grow together, the backends are. +3. **Errors and retries** — separates Envoy's own 503s from backend failures, and + shows whether retries are amplifying load on a struggling upstream. +4. **Upstream health and load balancing** — shrinking membership and outlier + ejections concentrate the same traffic on fewer hosts, which is a common cause + of pressure that looks like a traffic spike. *LB panic mode* firing means too + few healthy hosts remain and Envoy is balancing across unhealthy ones too. +5. **Envoy process health** — memory, watchdog misses (the clearest CPU-starvation + signal: a blocked event loop stalls every request on that worker), restarts and + the xDS control plane connection. + +### Assumptions + +- Envoy exposes stats via the Prometheus stats sink (`/stats/prometheus`), giving + the standard `envoy_` metric names and the `envoy_cluster_name`, + `envoy_response_code_class` and `envoy_http_conn_manager_prefix` labels. +- Two extra variables beyond the ones listed above: + - **upstream**: filters by `envoy_cluster_name` (the Envoy upstream cluster). + Distinct from **cluster**, which stays the NetBird deployment cluster. + - **host**: the Envoy instance, resolved from `envoy_server_live`. +- Circuit breaker panels read the `default` priority + (`envoy_cluster_circuit_breakers_default_*`). Add the `high` priority series if + your routes use it. +- Overload manager panels assume the default resource monitor names + (`fixed_heap`, `global_downstream_max_connections`). They stay empty if the + overload manager is not configured — worth configuring, since it is what keeps + Envoy from OOMing under exactly this kind of pressure.