Fall back to HTTP/1.1 when an upstream cannot serve the h2 it negotiated

ForceAttemptHTTP2 only puts h2 in the ALPN offer — the upstream still
picks — so "auto" already meant "whatever the upstream chose". What ALPN
cannot express is an upstream that selects h2 and then fails to speak it,
which is the case the setting was added for: today that leaves the
operator pinning every upstream to 1.1 to work around one broken backend.

Auto now completes itself. The first h2-level failure for a host pins
that host to an HTTP/1.1-only clone of its transport for 10 minutes and
retries the request there when the body can be replayed, so a broken
backend costs one failed h2 attempt instead of a configuration change.
The pin is per upstream host, so one broken backend does not drop the
others, and it expires so a fixed backend returns to h2 on its own.

Only h2 framing errors trigger it: a dial, TLS or context error says
nothing about the protocol and retrying it over HTTP/1.1 would fix
nothing. The explicit values stay absolute — "2" never downgrades.

Pinning HTTP/1.1 now also strips h2 from the ALPN offer. Configuring h2
makes net/http append it to the transport's TLSClientConfig, so a clone
taken from a transport that already served a request would otherwise
advertise a protocol the clone refuses to speak, and the reply would come
back as h2 frames parsed as an HTTP/1.1 message.

proxy/internal/roundtrip/multi.go

@@ -26,8 +26,8 @@ import (
 // branch at all), construct the MultiTransport via NewDirectOnly.
 type MultiTransport struct {
 	embedded http.RoundTripper
-	direct   *http.Transport
-	insecure *http.Transport
+	direct   *upstreamTransport
+	insecure *upstreamTransport
 }
 
 // errNoEmbeddedTransport is returned when a request reaches the
@@ -64,15 +64,13 @@ func NewMultiTransport(embedded http.RoundTripper, logger *log.Logger) *MultiTra
 		ReadBufferSize:        cfg.readBufferSize,
 		DisableCompression:    cfg.disableCompression,
 	}
-	applyUpstreamHTTPVersion(direct, cfg.upstreamHTTPVersion)
-
 	insecure := direct.Clone()
 	insecure.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // matches the embedded NetBird transport's per-target opt-in
 
 	return &MultiTransport{
 		embedded: embedded,
-		direct:   direct,
-		insecure: insecure,
+		direct:   newUpstreamTransport(direct, cfg.upstreamHTTPVersion, logger),
+		insecure: newUpstreamTransport(insecure, cfg.upstreamHTTPVersion, logger),
 	}
 }
 

proxy/internal/roundtrip/multi_test.go

@@ -76,13 +76,13 @@ func TestMultiTransport_AppliesEnvOverridesToDirect(t *testing.T) {
 
 	mt := NewMultiTransport(&stubRoundTripper{body: "embedded"}, nil)
 
-	assert.Equal(t, 42, mt.direct.MaxIdleConns,
+	assert.Equal(t, 42, mt.direct.primary.MaxIdleConns,
 		"NB_PROXY_MAX_IDLE_CONNS must propagate to the direct transport")
-	assert.Equal(t, 11*time.Second, mt.direct.IdleConnTimeout,
+	assert.Equal(t, 11*time.Second, mt.direct.primary.IdleConnTimeout,
 		"NB_PROXY_IDLE_CONN_TIMEOUT must propagate to the direct transport")
-	assert.Equal(t, 7*time.Second, mt.direct.TLSHandshakeTimeout,
+	assert.Equal(t, 7*time.Second, mt.direct.primary.TLSHandshakeTimeout,
 		"NB_PROXY_TLS_HANDSHAKE_TIMEOUT must propagate to the direct transport")
-	assert.Equal(t, 42, mt.insecure.MaxIdleConns,
+	assert.Equal(t, 42, mt.insecure.primary.MaxIdleConns,
 		"env tuning must also apply to the insecure-skip-verify direct transport")
 }
 

proxy/internal/roundtrip/netbird.go

@@ -82,10 +82,10 @@ type serviceNotification struct {
 // clientEntry holds an embedded NetBird client and tracks which services use it.
 type clientEntry struct {
 	client    *embed.Client
-	transport *http.Transport
+	transport *upstreamTransport
 	// insecureTransport is a clone of transport with TLS verification disabled,
 	// used when per-target skip_tls_verify is set.
-	insecureTransport *http.Transport
+	insecureTransport *upstreamTransport
 	services          map[ServiceKey]serviceInfo
 	createdAt         time.Time
 	started           bool
@@ -425,16 +425,14 @@ func (n *NetBird) createClientEntry(ctx context.Context, accountID types.Account
 		ReadBufferSize:        n.transportCfg.readBufferSize,
 		DisableCompression:    n.transportCfg.disableCompression,
 	}
-	applyUpstreamHTTPVersion(transport, n.transportCfg.upstreamHTTPVersion)
-
 	insecureTransport := transport.Clone()
 	insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec
 
 	return &clientEntry{
 		client:            client,
 		services:          map[ServiceKey]serviceInfo{key: si},
-		transport:         transport,
-		insecureTransport: insecureTransport,
+		transport:         newUpstreamTransport(transport, n.transportCfg.upstreamHTTPVersion, n.logger),
+		insecureTransport: newUpstreamTransport(insecureTransport, n.transportCfg.upstreamHTTPVersion, n.logger),
 		createdAt:         time.Now(),
 		started:           false,
 		inflightMap:       make(map[backendKey]chan struct{}),

proxy/internal/roundtrip/transport.go

@@ -34,13 +34,19 @@ const (
 type upstreamHTTPVersion string
 
 const (
-	// upstreamHTTPAuto leaves the choice to the proxy's own default.
-	// This is the only value whose meaning tracks that default.
+	// upstreamHTTPAuto leaves the choice to the upstream: h2 is offered
+	// alongside http/1.1 in the TLS handshake and the upstream picks.
+	// An upstream that picks h2 and then fails to serve it is moved to
+	// HTTP/1.1 on its own (see upstreamTransport), which is the part
+	// ALPN cannot express. This is the only value whose meaning tracks
+	// the proxy's default.
 	upstreamHTTPAuto upstreamHTTPVersion = "auto"
 	// upstreamHTTP11 never offers h2, so the upstream sees HTTP/1.1.
 	upstreamHTTP11 upstreamHTTPVersion = "1.1"
-	// upstreamHTTP2 offers h2 in the TLS handshake. Cleartext upstreams
-	// stay on HTTP/1.1 regardless: the proxy speaks no h2c.
+	// upstreamHTTP2 offers h2 in the TLS handshake and keeps it there:
+	// an upstream that negotiates h2 and then breaks is never moved to
+	// HTTP/1.1. Cleartext upstreams stay on HTTP/1.1 regardless: the
+	// proxy speaks no h2c.
 	upstreamHTTP2 upstreamHTTPVersion = "2"
 )
 
@@ -59,8 +65,9 @@ type transportConfig struct {
 	// maxInflight limits per-backend concurrent requests. 0 means unlimited.
 	maxInflight int
 	// upstreamHTTPVersion selects the HTTP version used towards HTTPS
-	// upstreams, for backends whose h2 support is advertised but
-	// unusable.
+	// upstreams. The default negotiates it with each upstream; the
+	// explicit values are for backends whose advertised h2 support is
+	// unusable and whose failure mode the negotiation cannot see.
 	upstreamHTTPVersion upstreamHTTPVersion
 }
 
@@ -134,10 +141,12 @@ func loadTransportConfig(logger *log.Logger) transportConfig {
 	return cfg
 }
 
-// applyUpstreamHTTPVersion configures t for the requested HTTP version.
-// It is the single place that decides what "auto" means, so changing the
-// proxy's default only touches this function and leaves every explicit
-// operator setting intact.
+// applyUpstreamHTTPVersion configures t's ALPN offer for the requested
+// HTTP version. It is the single place that decides which protocols a
+// transport offers, so changing the proxy's default only touches this
+// function and leaves every explicit operator setting intact. What
+// happens when a negotiated h2 upstream then fails belongs to
+// upstreamTransport, which owns the runtime half of "auto".
 //
 // HTTP/1.1 is pinned by clearing ForceAttemptHTTP2 and installing an
 // empty TLSNextProto, which disables h2 regardless of how the transport
@@ -148,11 +157,43 @@ func applyUpstreamHTTPVersion(t *http.Transport, version upstreamHTTPVersion) {
 	if version == upstreamHTTP11 {
 		t.ForceAttemptHTTP2 = false
 		t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
+		t.TLSClientConfig = withoutHTTP2ALPN(t.TLSClientConfig)
 		return
 	}
 	t.ForceAttemptHTTP2 = true
 }
 
+// withoutHTTP2ALPN drops h2 from the ALPN offer. Configuring h2 makes
+// net/http append h2 to the transport's TLSClientConfig, so a transport
+// cloned from one that already served a request carries that offer with
+// it. Left in place, the upstream would select a protocol this
+// transport then refuses to speak, and the response would come back as
+// h2 frames parsed as an HTTP/1.1 message.
+func withoutHTTP2ALPN(cfg *tls.Config) *tls.Config {
+	// A nil config offers no ALPN at all, which is already HTTP/1.1.
+	if cfg == nil || len(cfg.NextProtos) == 0 {
+		return cfg
+	}
+
+	protos := make([]string, 0, len(cfg.NextProtos))
+	for _, proto := range cfg.NextProtos {
+		if proto == "h2" {
+			continue
+		}
+		protos = append(protos, proto)
+	}
+	if len(protos) == len(cfg.NextProtos) {
+		return cfg
+	}
+
+	// Clone rather than edit in place: the caller may share this config
+	// with the transport it was cloned from.
+	stripped := cfg.Clone()
+	stripped.NextProtos = protos
+
+	return stripped
+}
+
 // envUpstreamHTTPVersion reads an upstream HTTP version from the
 // environment. An unrecognised value warns and leaves the default in
 // place rather than guessing at the operator's intent.

proxy/internal/roundtrip/upstream.go

@@ -0,0 +1,239 @@
+package roundtrip
+
+import (
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// upstreamDowngradeTTL is how long an upstream stays pinned to HTTP/1.1
+// after it proved it cannot serve the h2 it advertised. Bounded rather
+// than permanent so a fixed or replaced backend returns to h2 without
+// restarting the proxy.
+const upstreamDowngradeTTL = 10 * time.Minute
+
+// upstreamTransport carries requests to a single upstream family (one
+// TLS configuration) and implements what upstreamHTTPAuto means.
+//
+// ALPN already lets the upstream pick the protocol: primary offers both
+// h2 and http/1.1 and the server chooses. What ALPN cannot express is
+// an upstream that selects h2 and then fails to speak it — the case
+// this type handles. The first h2-level failure for a host pins that
+// host to fallback, an HTTP/1.1-only clone of primary, and the request
+// is retried there when it can be replayed.
+//
+// The downgrade is per upstream host, not per transport: one broken
+// backend must not drop every other backend to HTTP/1.1.
+type upstreamTransport struct {
+	// primary is the configured transport: h2 offered in ALPN for
+	// upstreamHTTPAuto and upstreamHTTP2, HTTP/1.1-only for
+	// upstreamHTTP11.
+	primary *http.Transport
+	// version decides whether a downgrade may happen at all. Only
+	// upstreamHTTPAuto downgrades; the explicit values are absolute.
+	version upstreamHTTPVersion
+	logger  *log.Logger
+
+	// fallbackMu guards the lazy fallback clone: most deployments never
+	// hit a broken h2 upstream and should not pay for a second
+	// connection pool.
+	fallbackMu sync.Mutex
+	fallback   *http.Transport
+
+	mu sync.RWMutex
+	// downgraded maps an upstream host to the time its HTTP/1.1 pin
+	// expires.
+	downgraded map[string]time.Time
+}
+
+// newUpstreamTransport wraps base for the requested HTTP version. base
+// must not be used directly afterwards: the wrapper owns it, including
+// its connection pool.
+func newUpstreamTransport(base *http.Transport, version upstreamHTTPVersion, logger *log.Logger) *upstreamTransport {
+	if logger == nil {
+		logger = log.StandardLogger()
+	}
+	applyUpstreamHTTPVersion(base, version)
+
+	return &upstreamTransport{
+		primary:    base,
+		version:    version,
+		logger:     logger,
+		downgraded: make(map[string]time.Time),
+	}
+}
+
+// RoundTrip implements http.RoundTripper.
+func (t *upstreamTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	if !t.mayDowngrade(req) {
+		return t.primary.RoundTrip(req)
+	}
+
+	host := req.URL.Host
+	if t.isDowngraded(host) {
+		return t.http1().RoundTrip(req)
+	}
+
+	resp, err := t.primary.RoundTrip(req)
+	if err == nil || !isHTTP2ProtocolError(err) {
+		return resp, err
+	}
+
+	t.markDowngraded(host)
+
+	retry, ok := replayable(req)
+	if !ok {
+		// The body is already consumed and cannot be regenerated, so
+		// this request fails. The host is pinned either way, so the
+		// next one goes out over HTTP/1.1.
+		return nil, err
+	}
+	return t.http1().RoundTrip(retry)
+}
+
+// CloseIdleConnections closes idle connections on both pools.
+func (t *upstreamTransport) CloseIdleConnections() {
+	t.primary.CloseIdleConnections()
+	if fallback := t.existingHTTP1(); fallback != nil {
+		fallback.CloseIdleConnections()
+	}
+}
+
+// mayDowngrade reports whether a failed request is a downgrade
+// candidate. Only upstreamHTTPAuto downgrades, and only for TLS
+// upstreams: the proxy speaks no h2c, so a cleartext upstream is
+// already on HTTP/1.1 and an error there says nothing about h2.
+func (t *upstreamTransport) mayDowngrade(req *http.Request) bool {
+	return t.version == upstreamHTTPAuto && req.URL != nil && req.URL.Scheme == "https"
+}
+
+func (t *upstreamTransport) isDowngraded(host string) bool {
+	t.mu.RLock()
+	expiry, ok := t.downgraded[host]
+	t.mu.RUnlock()
+
+	if !ok {
+		return false
+	}
+	if time.Now().Before(expiry) {
+		return true
+	}
+
+	t.mu.Lock()
+	// Re-check under the write lock: a concurrent request may have
+	// re-pinned the host after the read above.
+	if expiry, ok := t.downgraded[host]; ok && !time.Now().Before(expiry) {
+		delete(t.downgraded, host)
+	}
+	t.mu.Unlock()
+
+	return false
+}
+
+func (t *upstreamTransport) markDowngraded(host string) {
+	now := time.Now()
+
+	t.mu.Lock()
+	_, pinned := t.downgraded[host]
+	t.downgraded[host] = now.Add(upstreamDowngradeTTL)
+	for h, expiry := range t.downgraded {
+		if !now.Before(expiry) {
+			delete(t.downgraded, h)
+		}
+	}
+	t.mu.Unlock()
+
+	if !pinned {
+		t.logger.WithField("upstream", host).
+			Warnf("upstream negotiated HTTP/2 but failed to serve it, using HTTP/1.1 for the next %s (set %s=1.1 to pin it)",
+				upstreamDowngradeTTL, EnvUpstreamHTTPVersion)
+	}
+}
+
+// http1 returns the HTTP/1.1-only clone, creating it on first use.
+func (t *upstreamTransport) http1() *http.Transport {
+	t.fallbackMu.Lock()
+	defer t.fallbackMu.Unlock()
+
+	if t.fallback == nil {
+		fallback := t.primary.Clone()
+		applyUpstreamHTTPVersion(fallback, upstreamHTTP11)
+		t.fallback = fallback
+	}
+
+	return t.fallback
+}
+
+// existingHTTP1 returns the fallback transport only if it was already
+// created, so housekeeping never allocates a second connection pool for
+// an upstream that never needed one.
+func (t *upstreamTransport) existingHTTP1() *http.Transport {
+	t.fallbackMu.Lock()
+	defer t.fallbackMu.Unlock()
+
+	return t.fallback
+}
+
+// replayable returns a request that can be sent a second time, or
+// ok=false when the body is gone. A RoundTripper consumes and closes
+// the body it was given, so a retry needs either no body at all or
+// GetBody to produce a fresh one.
+func replayable(req *http.Request) (*http.Request, bool) {
+	if req.Body == nil || req.Body == http.NoBody {
+		return req, true
+	}
+	if req.GetBody == nil {
+		return nil, false
+	}
+
+	body, err := req.GetBody()
+	if err != nil {
+		return nil, false
+	}
+
+	retry := req.Clone(req.Context())
+	retry.Body = body
+
+	return retry, true
+}
+
+// http2ErrorMarkers are the substrings that identify an HTTP/2 protocol
+// failure. net/http bundles its own private copy of the http2 package,
+// so its errors cannot be matched by type from here: http2.StreamError
+// and friends in x/net are different types from the ones a
+// bundled-h2 transport returns. The strings below are the formats those
+// bundled errors print, and they are specific to h2 framing — a
+// downgrade must never be triggered by an ordinary network or TLS
+// error, which retrying on HTTP/1.1 would not fix.
+var http2ErrorMarkers = []string{
+	// Transport-level h2 failures, e.g.
+	// "http2: server sent GOAWAY and closed the connection".
+	"http2:",
+	// http2.StreamError, e.g. "stream error: stream ID 1; PROTOCOL_ERROR".
+	"stream error: stream ID",
+	// http2.ConnectionError, e.g. "connection error: PROTOCOL_ERROR".
+	"connection error: ",
+	// The GOAWAY code an upstream sends to say the request must be
+	// retried over HTTP/1.1.
+	"HTTP_1_1_REQUIRED",
+}
+
+// isHTTP2ProtocolError reports whether err says the upstream cannot
+// serve the h2 it negotiated.
+func isHTTP2ProtocolError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	msg := err.Error()
+	for _, marker := range http2ErrorMarkers {
+		if strings.Contains(msg, marker) {
+			return true
+		}
+	}
+
+	return false
+}

proxy/internal/roundtrip/upstream_test.go

@@ -0,0 +1,323 @@
+package roundtrip
+
+import (
+	"bufio"
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"errors"
+	"io"
+	"math/big"
+	"net"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/http2"
+)
+
+// TestUpstreamTransport_AutoFallsBackOnBrokenHTTP2 covers the case ALPN
+// cannot express: the upstream advertises h2, picks it, and then cannot
+// serve it. The request must still succeed, over HTTP/1.1, and the
+// upstream must stay on HTTP/1.1 for the requests that follow.
+func TestUpstreamTransport_AutoFallsBackOnBrokenHTTP2(t *testing.T) {
+	t.Setenv(EnvUpstreamHTTPVersion, string(upstreamHTTPAuto))
+	srv := startBrokenHTTP2Server(t)
+
+	mt := NewDirectOnly(nil)
+	ctx := WithSkipTLSVerify(WithDirectUpstream(context.Background()))
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+srv.addr, nil)
+	require.NoError(t, err)
+
+	resp, err := mt.RoundTrip(req)
+	require.NoError(t, err, "a replayable request must be retried on HTTP/1.1 instead of failing")
+	body, err := io.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+	require.NoError(t, err)
+	assert.Equal(t, "HTTP/1.1", resp.Proto, "the retry must ride the HTTP/1.1 transport")
+	assert.Equal(t, "http/1.1", string(body), "the upstream must see an http/1.1 ALPN offer on the retry")
+
+	assert.True(t, mt.insecure.isDowngraded(srv.addr),
+		"the upstream must stay pinned to HTTP/1.1 after proving it cannot serve h2")
+
+	// The second request must not repeat the h2 attempt: the server
+	// counts h2 handshakes, so a repeat would show up here.
+	h2Attempts := srv.http2Handshakes()
+	req, err = http.NewRequestWithContext(ctx, http.MethodGet, "https://"+srv.addr, nil)
+	require.NoError(t, err)
+	resp, err = mt.RoundTrip(req)
+	require.NoError(t, err)
+	_ = resp.Body.Close()
+
+	assert.Equal(t, "HTTP/1.1", resp.Proto, "a pinned upstream must go straight to HTTP/1.1")
+	assert.Equal(t, h2Attempts, srv.http2Handshakes(),
+		"a pinned upstream must not be probed for h2 again until the pin expires")
+}
+
+// TestUpstreamTransport_ExplicitHTTP2NeverDowngrades pins the promise
+// that the explicit values are absolute: an operator who asked for h2
+// keeps h2, broken upstream or not.
+func TestUpstreamTransport_ExplicitHTTP2NeverDowngrades(t *testing.T) {
+	t.Setenv(EnvUpstreamHTTPVersion, string(upstreamHTTP2))
+	srv := startBrokenHTTP2Server(t)
+
+	mt := NewDirectOnly(nil)
+	ctx := WithSkipTLSVerify(WithDirectUpstream(context.Background()))
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+srv.addr, nil)
+	require.NoError(t, err)
+
+	resp, err := mt.RoundTrip(req)
+	if err == nil {
+		_ = resp.Body.Close()
+	}
+	require.Error(t, err, "NB_PROXY_UPSTREAM_HTTP_VERSION=2 must not fall back to HTTP/1.1")
+	assert.False(t, mt.insecure.isDowngraded(srv.addr), "an explicit version must never pin an upstream")
+}
+
+func TestUpstreamTransport_MayDowngrade(t *testing.T) {
+	tests := []struct {
+		name    string
+		version upstreamHTTPVersion
+		url     string
+		want    bool
+	}{
+		{name: "auto over TLS", version: upstreamHTTPAuto, url: "https://backend.invalid", want: true},
+		// The proxy speaks no h2c, so a cleartext upstream is already on
+		// HTTP/1.1 and its failures say nothing about h2.
+		{name: "auto cleartext", version: upstreamHTTPAuto, url: "http://backend.invalid", want: false},
+		{name: "explicit 1.1", version: upstreamHTTP11, url: "https://backend.invalid", want: false},
+		{name: "explicit 2", version: upstreamHTTP2, url: "https://backend.invalid", want: false},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			transport := newUpstreamTransport(&http.Transport{}, tc.version, nil)
+			req, err := http.NewRequest(http.MethodGet, tc.url, nil)
+			require.NoError(t, err)
+
+			assert.Equal(t, tc.want, transport.mayDowngrade(req))
+		})
+	}
+}
+
+func TestUpstreamTransport_DowngradeExpires(t *testing.T) {
+	transport := newUpstreamTransport(&http.Transport{}, upstreamHTTPAuto, nil)
+	transport.markDowngraded("backend.invalid:443")
+	require.True(t, transport.isDowngraded("backend.invalid:443"))
+
+	transport.mu.Lock()
+	transport.downgraded["backend.invalid:443"] = time.Now().Add(-time.Second)
+	transport.mu.Unlock()
+
+	assert.False(t, transport.isDowngraded("backend.invalid:443"),
+		"an expired pin must let the upstream be offered h2 again")
+	transport.mu.RLock()
+	_, stillTracked := transport.downgraded["backend.invalid:443"]
+	transport.mu.RUnlock()
+	assert.False(t, stillTracked, "an expired pin must not be kept around")
+}
+
+func TestUpstreamTransport_DowngradeIsPerUpstream(t *testing.T) {
+	transport := newUpstreamTransport(&http.Transport{}, upstreamHTTPAuto, nil)
+	transport.markDowngraded("broken.invalid:443")
+
+	assert.True(t, transport.isDowngraded("broken.invalid:443"))
+	assert.False(t, transport.isDowngraded("healthy.invalid:443"),
+		"one broken upstream must not drop the others to HTTP/1.1")
+}
+
+func TestIsHTTP2ProtocolError(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{
+			name: "goaway demanding http/1.1",
+			err:  errors.New(`http2: server sent GOAWAY and closed the connection; LastStreamID=0, ErrCode=HTTP_1_1_REQUIRED, debug=""`),
+			want: true,
+		},
+		{
+			name: "stream error",
+			err:  http2.StreamError{StreamID: 1, Code: http2.ErrCodeProtocol},
+			want: true,
+		},
+		{
+			name: "connection error",
+			err:  http2.ConnectionError(http2.ErrCodeProtocol),
+			want: true,
+		},
+		{
+			name: "wrapped h2 error",
+			err:  errors.New("Get \"https://backend.invalid\": http2: client connection lost"),
+			want: true,
+		},
+		// Retrying these on HTTP/1.1 fixes nothing, so they must never
+		// pin an upstream.
+		{name: "dial failure", err: errors.New("dial tcp 10.0.0.1:443: connect: connection refused"), want: false},
+		{name: "tls failure", err: errors.New("tls: failed to verify certificate: x509: certificate signed by unknown authority"), want: false},
+		{name: "context cancelled", err: context.Canceled, want: false},
+		{name: "eof", err: io.EOF, want: false},
+		{name: "nil", err: nil, want: false},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.want, isHTTP2ProtocolError(tc.err))
+		})
+	}
+}
+
+func TestReplayable(t *testing.T) {
+	t.Run("bodyless request", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodGet, "https://backend.invalid", nil)
+		require.NoError(t, err)
+
+		retry, ok := replayable(req)
+		require.True(t, ok)
+		assert.Same(t, req, retry, "a bodyless request needs no clone")
+	})
+
+	t.Run("request with GetBody", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodPost, "https://backend.invalid", strings.NewReader("payload"))
+		require.NoError(t, err)
+		// Consume the body the way a failed RoundTrip would.
+		_, err = io.ReadAll(req.Body)
+		require.NoError(t, err)
+
+		retry, ok := replayable(req)
+		require.True(t, ok)
+		body, err := io.ReadAll(retry.Body)
+		require.NoError(t, err)
+		assert.Equal(t, "payload", string(body), "the retry must carry a fresh copy of the body")
+	})
+
+	t.Run("streamed request", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodPost, "https://backend.invalid", io.NopCloser(strings.NewReader("payload")))
+		require.NoError(t, err)
+		require.Nil(t, req.GetBody, "an opaque reader must not get a GetBody")
+
+		_, ok := replayable(req)
+		assert.False(t, ok, "a body that cannot be regenerated must not be replayed")
+	})
+}
+
+// brokenHTTP2Server advertises h2 in ALPN, accepts it, and then refuses
+// to serve it — the upstream behaviour that motivated the fallback. Over
+// http/1.1 it answers normally, so a downgraded request succeeds.
+type brokenHTTP2Server struct {
+	addr string
+
+	handshakes chan struct{}
+}
+
+func (s *brokenHTTP2Server) http2Handshakes() int {
+	return len(s.handshakes)
+}
+
+func startBrokenHTTP2Server(t *testing.T) *brokenHTTP2Server {
+	t.Helper()
+
+	ln, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{
+		Certificates: []tls.Certificate{selfSignedCert(t)},
+		NextProtos:   []string{"h2", "http/1.1"},
+		MinVersion:   tls.VersionTLS12,
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = ln.Close() })
+
+	srv := &brokenHTTP2Server{
+		addr: ln.Addr().String(),
+		// Buffered well past what the test drives so a stuck server
+		// never blocks the accept loop.
+		handshakes: make(chan struct{}, 64),
+	}
+
+	go func() {
+		for {
+			conn, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			go srv.handle(conn)
+		}
+	}()
+
+	return srv
+}
+
+func (s *brokenHTTP2Server) handle(conn net.Conn) {
+	defer func() { _ = conn.Close() }()
+
+	tlsConn, ok := conn.(*tls.Conn)
+	if !ok {
+		return
+	}
+	if err := tlsConn.Handshake(); err != nil {
+		return
+	}
+
+	if tlsConn.ConnectionState().NegotiatedProtocol == "h2" {
+		select {
+		case s.handshakes <- struct{}{}:
+		default:
+		}
+		s.refuseHTTP2(tlsConn)
+		return
+	}
+
+	s.serveHTTP1(tlsConn)
+}
+
+// refuseHTTP2 completes just enough of the h2 handshake for the client
+// to accept the connection, then sends the GOAWAY an upstream uses to
+// say the request belongs on HTTP/1.1.
+func (s *brokenHTTP2Server) refuseHTTP2(conn net.Conn) {
+	framer := http2.NewFramer(conn, conn)
+	if err := framer.WriteSettings(); err != nil {
+		return
+	}
+	_ = framer.WriteGoAway(0, http2.ErrCodeHTTP11Required, nil)
+}
+
+// serveHTTP1 answers a single request with the protocol the upstream
+// saw, so the test can tell which transport carried it.
+func (s *brokenHTTP2Server) serveHTTP1(conn net.Conn) {
+	reader := bufio.NewReader(conn)
+	if _, err := http.ReadRequest(reader); err != nil {
+		return
+	}
+
+	const body = "http/1.1"
+	_, _ = io.WriteString(conn, "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 8\r\nConnection: close\r\n\r\n"+body)
+}
+
+func selfSignedCert(t *testing.T) tls.Certificate {
+	t.Helper()
+
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+
+	template := x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject:      pkix.Name{CommonName: "127.0.0.1"},
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(time.Hour),
+		KeyUsage:     x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		IPAddresses:  []net.IP{net.ParseIP("127.0.0.1")},
+		IsCA:         true,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
+	require.NoError(t, err)
+
+	return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
+}