diff --git a/proxy/internal/roundtrip/multi.go b/proxy/internal/roundtrip/multi.go index 3604e17d2..1abf54a8d 100644 --- a/proxy/internal/roundtrip/multi.go +++ b/proxy/internal/roundtrip/multi.go @@ -26,8 +26,8 @@ import ( // branch at all), construct the MultiTransport via NewDirectOnly. type MultiTransport struct { embedded http.RoundTripper - direct *http.Transport - insecure *http.Transport + direct *upstreamTransport + insecure *upstreamTransport } // errNoEmbeddedTransport is returned when a request reaches the @@ -64,15 +64,13 @@ func NewMultiTransport(embedded http.RoundTripper, logger *log.Logger) *MultiTra ReadBufferSize: cfg.readBufferSize, DisableCompression: cfg.disableCompression, } - applyUpstreamHTTPVersion(direct, cfg.upstreamHTTPVersion) - insecure := direct.Clone() insecure.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // matches the embedded NetBird transport's per-target opt-in return &MultiTransport{ embedded: embedded, - direct: direct, - insecure: insecure, + direct: newUpstreamTransport(direct, cfg.upstreamHTTPVersion, logger), + insecure: newUpstreamTransport(insecure, cfg.upstreamHTTPVersion, logger), } } diff --git a/proxy/internal/roundtrip/multi_test.go b/proxy/internal/roundtrip/multi_test.go index 6b1229a3a..7b8b50e81 100644 --- a/proxy/internal/roundtrip/multi_test.go +++ b/proxy/internal/roundtrip/multi_test.go @@ -76,13 +76,13 @@ func TestMultiTransport_AppliesEnvOverridesToDirect(t *testing.T) { mt := NewMultiTransport(&stubRoundTripper{body: "embedded"}, nil) - assert.Equal(t, 42, mt.direct.MaxIdleConns, + assert.Equal(t, 42, mt.direct.primary.MaxIdleConns, "NB_PROXY_MAX_IDLE_CONNS must propagate to the direct transport") - assert.Equal(t, 11*time.Second, mt.direct.IdleConnTimeout, + assert.Equal(t, 11*time.Second, mt.direct.primary.IdleConnTimeout, "NB_PROXY_IDLE_CONN_TIMEOUT must propagate to the direct transport") - assert.Equal(t, 7*time.Second, mt.direct.TLSHandshakeTimeout, + assert.Equal(t, 7*time.Second, mt.direct.primary.TLSHandshakeTimeout, "NB_PROXY_TLS_HANDSHAKE_TIMEOUT must propagate to the direct transport") - assert.Equal(t, 42, mt.insecure.MaxIdleConns, + assert.Equal(t, 42, mt.insecure.primary.MaxIdleConns, "env tuning must also apply to the insecure-skip-verify direct transport") } diff --git a/proxy/internal/roundtrip/netbird.go b/proxy/internal/roundtrip/netbird.go index 67e3ce039..d7b464182 100644 --- a/proxy/internal/roundtrip/netbird.go +++ b/proxy/internal/roundtrip/netbird.go @@ -82,10 +82,10 @@ type serviceNotification struct { // clientEntry holds an embedded NetBird client and tracks which services use it. type clientEntry struct { client *embed.Client - transport *http.Transport + transport *upstreamTransport // insecureTransport is a clone of transport with TLS verification disabled, // used when per-target skip_tls_verify is set. - insecureTransport *http.Transport + insecureTransport *upstreamTransport services map[ServiceKey]serviceInfo createdAt time.Time started bool @@ -425,16 +425,14 @@ func (n *NetBird) createClientEntry(ctx context.Context, accountID types.Account ReadBufferSize: n.transportCfg.readBufferSize, DisableCompression: n.transportCfg.disableCompression, } - applyUpstreamHTTPVersion(transport, n.transportCfg.upstreamHTTPVersion) - insecureTransport := transport.Clone() insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec return &clientEntry{ client: client, services: map[ServiceKey]serviceInfo{key: si}, - transport: transport, - insecureTransport: insecureTransport, + transport: newUpstreamTransport(transport, n.transportCfg.upstreamHTTPVersion, n.logger), + insecureTransport: newUpstreamTransport(insecureTransport, n.transportCfg.upstreamHTTPVersion, n.logger), createdAt: time.Now(), started: false, inflightMap: make(map[backendKey]chan struct{}), diff --git a/proxy/internal/roundtrip/transport.go b/proxy/internal/roundtrip/transport.go index 093d8b3f3..9e872e447 100644 --- a/proxy/internal/roundtrip/transport.go +++ b/proxy/internal/roundtrip/transport.go @@ -34,13 +34,19 @@ const ( type upstreamHTTPVersion string const ( - // upstreamHTTPAuto leaves the choice to the proxy's own default. - // This is the only value whose meaning tracks that default. + // upstreamHTTPAuto leaves the choice to the upstream: h2 is offered + // alongside http/1.1 in the TLS handshake and the upstream picks. + // An upstream that picks h2 and then fails to serve it is moved to + // HTTP/1.1 on its own (see upstreamTransport), which is the part + // ALPN cannot express. This is the only value whose meaning tracks + // the proxy's default. upstreamHTTPAuto upstreamHTTPVersion = "auto" // upstreamHTTP11 never offers h2, so the upstream sees HTTP/1.1. upstreamHTTP11 upstreamHTTPVersion = "1.1" - // upstreamHTTP2 offers h2 in the TLS handshake. Cleartext upstreams - // stay on HTTP/1.1 regardless: the proxy speaks no h2c. + // upstreamHTTP2 offers h2 in the TLS handshake and keeps it there: + // an upstream that negotiates h2 and then breaks is never moved to + // HTTP/1.1. Cleartext upstreams stay on HTTP/1.1 regardless: the + // proxy speaks no h2c. upstreamHTTP2 upstreamHTTPVersion = "2" ) @@ -59,8 +65,9 @@ type transportConfig struct { // maxInflight limits per-backend concurrent requests. 0 means unlimited. maxInflight int // upstreamHTTPVersion selects the HTTP version used towards HTTPS - // upstreams, for backends whose h2 support is advertised but - // unusable. + // upstreams. The default negotiates it with each upstream; the + // explicit values are for backends whose advertised h2 support is + // unusable and whose failure mode the negotiation cannot see. upstreamHTTPVersion upstreamHTTPVersion } @@ -134,10 +141,12 @@ func loadTransportConfig(logger *log.Logger) transportConfig { return cfg } -// applyUpstreamHTTPVersion configures t for the requested HTTP version. -// It is the single place that decides what "auto" means, so changing the -// proxy's default only touches this function and leaves every explicit -// operator setting intact. +// applyUpstreamHTTPVersion configures t's ALPN offer for the requested +// HTTP version. It is the single place that decides which protocols a +// transport offers, so changing the proxy's default only touches this +// function and leaves every explicit operator setting intact. What +// happens when a negotiated h2 upstream then fails belongs to +// upstreamTransport, which owns the runtime half of "auto". // // HTTP/1.1 is pinned by clearing ForceAttemptHTTP2 and installing an // empty TLSNextProto, which disables h2 regardless of how the transport @@ -148,11 +157,43 @@ func applyUpstreamHTTPVersion(t *http.Transport, version upstreamHTTPVersion) { if version == upstreamHTTP11 { t.ForceAttemptHTTP2 = false t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{} + t.TLSClientConfig = withoutHTTP2ALPN(t.TLSClientConfig) return } t.ForceAttemptHTTP2 = true } +// withoutHTTP2ALPN drops h2 from the ALPN offer. Configuring h2 makes +// net/http append h2 to the transport's TLSClientConfig, so a transport +// cloned from one that already served a request carries that offer with +// it. Left in place, the upstream would select a protocol this +// transport then refuses to speak, and the response would come back as +// h2 frames parsed as an HTTP/1.1 message. +func withoutHTTP2ALPN(cfg *tls.Config) *tls.Config { + // A nil config offers no ALPN at all, which is already HTTP/1.1. + if cfg == nil || len(cfg.NextProtos) == 0 { + return cfg + } + + protos := make([]string, 0, len(cfg.NextProtos)) + for _, proto := range cfg.NextProtos { + if proto == "h2" { + continue + } + protos = append(protos, proto) + } + if len(protos) == len(cfg.NextProtos) { + return cfg + } + + // Clone rather than edit in place: the caller may share this config + // with the transport it was cloned from. + stripped := cfg.Clone() + stripped.NextProtos = protos + + return stripped +} + // envUpstreamHTTPVersion reads an upstream HTTP version from the // environment. An unrecognised value warns and leaves the default in // place rather than guessing at the operator's intent. diff --git a/proxy/internal/roundtrip/upstream.go b/proxy/internal/roundtrip/upstream.go new file mode 100644 index 000000000..8f582b0d2 --- /dev/null +++ b/proxy/internal/roundtrip/upstream.go @@ -0,0 +1,239 @@ +package roundtrip + +import ( + "net/http" + "strings" + "sync" + "time" + + log "github.com/sirupsen/logrus" +) + +// upstreamDowngradeTTL is how long an upstream stays pinned to HTTP/1.1 +// after it proved it cannot serve the h2 it advertised. Bounded rather +// than permanent so a fixed or replaced backend returns to h2 without +// restarting the proxy. +const upstreamDowngradeTTL = 10 * time.Minute + +// upstreamTransport carries requests to a single upstream family (one +// TLS configuration) and implements what upstreamHTTPAuto means. +// +// ALPN already lets the upstream pick the protocol: primary offers both +// h2 and http/1.1 and the server chooses. What ALPN cannot express is +// an upstream that selects h2 and then fails to speak it — the case +// this type handles. The first h2-level failure for a host pins that +// host to fallback, an HTTP/1.1-only clone of primary, and the request +// is retried there when it can be replayed. +// +// The downgrade is per upstream host, not per transport: one broken +// backend must not drop every other backend to HTTP/1.1. +type upstreamTransport struct { + // primary is the configured transport: h2 offered in ALPN for + // upstreamHTTPAuto and upstreamHTTP2, HTTP/1.1-only for + // upstreamHTTP11. + primary *http.Transport + // version decides whether a downgrade may happen at all. Only + // upstreamHTTPAuto downgrades; the explicit values are absolute. + version upstreamHTTPVersion + logger *log.Logger + + // fallbackMu guards the lazy fallback clone: most deployments never + // hit a broken h2 upstream and should not pay for a second + // connection pool. + fallbackMu sync.Mutex + fallback *http.Transport + + mu sync.RWMutex + // downgraded maps an upstream host to the time its HTTP/1.1 pin + // expires. + downgraded map[string]time.Time +} + +// newUpstreamTransport wraps base for the requested HTTP version. base +// must not be used directly afterwards: the wrapper owns it, including +// its connection pool. +func newUpstreamTransport(base *http.Transport, version upstreamHTTPVersion, logger *log.Logger) *upstreamTransport { + if logger == nil { + logger = log.StandardLogger() + } + applyUpstreamHTTPVersion(base, version) + + return &upstreamTransport{ + primary: base, + version: version, + logger: logger, + downgraded: make(map[string]time.Time), + } +} + +// RoundTrip implements http.RoundTripper. +func (t *upstreamTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if !t.mayDowngrade(req) { + return t.primary.RoundTrip(req) + } + + host := req.URL.Host + if t.isDowngraded(host) { + return t.http1().RoundTrip(req) + } + + resp, err := t.primary.RoundTrip(req) + if err == nil || !isHTTP2ProtocolError(err) { + return resp, err + } + + t.markDowngraded(host) + + retry, ok := replayable(req) + if !ok { + // The body is already consumed and cannot be regenerated, so + // this request fails. The host is pinned either way, so the + // next one goes out over HTTP/1.1. + return nil, err + } + return t.http1().RoundTrip(retry) +} + +// CloseIdleConnections closes idle connections on both pools. +func (t *upstreamTransport) CloseIdleConnections() { + t.primary.CloseIdleConnections() + if fallback := t.existingHTTP1(); fallback != nil { + fallback.CloseIdleConnections() + } +} + +// mayDowngrade reports whether a failed request is a downgrade +// candidate. Only upstreamHTTPAuto downgrades, and only for TLS +// upstreams: the proxy speaks no h2c, so a cleartext upstream is +// already on HTTP/1.1 and an error there says nothing about h2. +func (t *upstreamTransport) mayDowngrade(req *http.Request) bool { + return t.version == upstreamHTTPAuto && req.URL != nil && req.URL.Scheme == "https" +} + +func (t *upstreamTransport) isDowngraded(host string) bool { + t.mu.RLock() + expiry, ok := t.downgraded[host] + t.mu.RUnlock() + + if !ok { + return false + } + if time.Now().Before(expiry) { + return true + } + + t.mu.Lock() + // Re-check under the write lock: a concurrent request may have + // re-pinned the host after the read above. + if expiry, ok := t.downgraded[host]; ok && !time.Now().Before(expiry) { + delete(t.downgraded, host) + } + t.mu.Unlock() + + return false +} + +func (t *upstreamTransport) markDowngraded(host string) { + now := time.Now() + + t.mu.Lock() + _, pinned := t.downgraded[host] + t.downgraded[host] = now.Add(upstreamDowngradeTTL) + for h, expiry := range t.downgraded { + if !now.Before(expiry) { + delete(t.downgraded, h) + } + } + t.mu.Unlock() + + if !pinned { + t.logger.WithField("upstream", host). + Warnf("upstream negotiated HTTP/2 but failed to serve it, using HTTP/1.1 for the next %s (set %s=1.1 to pin it)", + upstreamDowngradeTTL, EnvUpstreamHTTPVersion) + } +} + +// http1 returns the HTTP/1.1-only clone, creating it on first use. +func (t *upstreamTransport) http1() *http.Transport { + t.fallbackMu.Lock() + defer t.fallbackMu.Unlock() + + if t.fallback == nil { + fallback := t.primary.Clone() + applyUpstreamHTTPVersion(fallback, upstreamHTTP11) + t.fallback = fallback + } + + return t.fallback +} + +// existingHTTP1 returns the fallback transport only if it was already +// created, so housekeeping never allocates a second connection pool for +// an upstream that never needed one. +func (t *upstreamTransport) existingHTTP1() *http.Transport { + t.fallbackMu.Lock() + defer t.fallbackMu.Unlock() + + return t.fallback +} + +// replayable returns a request that can be sent a second time, or +// ok=false when the body is gone. A RoundTripper consumes and closes +// the body it was given, so a retry needs either no body at all or +// GetBody to produce a fresh one. +func replayable(req *http.Request) (*http.Request, bool) { + if req.Body == nil || req.Body == http.NoBody { + return req, true + } + if req.GetBody == nil { + return nil, false + } + + body, err := req.GetBody() + if err != nil { + return nil, false + } + + retry := req.Clone(req.Context()) + retry.Body = body + + return retry, true +} + +// http2ErrorMarkers are the substrings that identify an HTTP/2 protocol +// failure. net/http bundles its own private copy of the http2 package, +// so its errors cannot be matched by type from here: http2.StreamError +// and friends in x/net are different types from the ones a +// bundled-h2 transport returns. The strings below are the formats those +// bundled errors print, and they are specific to h2 framing — a +// downgrade must never be triggered by an ordinary network or TLS +// error, which retrying on HTTP/1.1 would not fix. +var http2ErrorMarkers = []string{ + // Transport-level h2 failures, e.g. + // "http2: server sent GOAWAY and closed the connection". + "http2:", + // http2.StreamError, e.g. "stream error: stream ID 1; PROTOCOL_ERROR". + "stream error: stream ID", + // http2.ConnectionError, e.g. "connection error: PROTOCOL_ERROR". + "connection error: ", + // The GOAWAY code an upstream sends to say the request must be + // retried over HTTP/1.1. + "HTTP_1_1_REQUIRED", +} + +// isHTTP2ProtocolError reports whether err says the upstream cannot +// serve the h2 it negotiated. +func isHTTP2ProtocolError(err error) bool { + if err == nil { + return false + } + + msg := err.Error() + for _, marker := range http2ErrorMarkers { + if strings.Contains(msg, marker) { + return true + } + } + + return false +} diff --git a/proxy/internal/roundtrip/upstream_test.go b/proxy/internal/roundtrip/upstream_test.go new file mode 100644 index 000000000..cc8b2ab33 --- /dev/null +++ b/proxy/internal/roundtrip/upstream_test.go @@ -0,0 +1,323 @@ +package roundtrip + +import ( + "bufio" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "io" + "math/big" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/http2" +) + +// TestUpstreamTransport_AutoFallsBackOnBrokenHTTP2 covers the case ALPN +// cannot express: the upstream advertises h2, picks it, and then cannot +// serve it. The request must still succeed, over HTTP/1.1, and the +// upstream must stay on HTTP/1.1 for the requests that follow. +func TestUpstreamTransport_AutoFallsBackOnBrokenHTTP2(t *testing.T) { + t.Setenv(EnvUpstreamHTTPVersion, string(upstreamHTTPAuto)) + srv := startBrokenHTTP2Server(t) + + mt := NewDirectOnly(nil) + ctx := WithSkipTLSVerify(WithDirectUpstream(context.Background())) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+srv.addr, nil) + require.NoError(t, err) + + resp, err := mt.RoundTrip(req) + require.NoError(t, err, "a replayable request must be retried on HTTP/1.1 instead of failing") + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + require.NoError(t, err) + assert.Equal(t, "HTTP/1.1", resp.Proto, "the retry must ride the HTTP/1.1 transport") + assert.Equal(t, "http/1.1", string(body), "the upstream must see an http/1.1 ALPN offer on the retry") + + assert.True(t, mt.insecure.isDowngraded(srv.addr), + "the upstream must stay pinned to HTTP/1.1 after proving it cannot serve h2") + + // The second request must not repeat the h2 attempt: the server + // counts h2 handshakes, so a repeat would show up here. + h2Attempts := srv.http2Handshakes() + req, err = http.NewRequestWithContext(ctx, http.MethodGet, "https://"+srv.addr, nil) + require.NoError(t, err) + resp, err = mt.RoundTrip(req) + require.NoError(t, err) + _ = resp.Body.Close() + + assert.Equal(t, "HTTP/1.1", resp.Proto, "a pinned upstream must go straight to HTTP/1.1") + assert.Equal(t, h2Attempts, srv.http2Handshakes(), + "a pinned upstream must not be probed for h2 again until the pin expires") +} + +// TestUpstreamTransport_ExplicitHTTP2NeverDowngrades pins the promise +// that the explicit values are absolute: an operator who asked for h2 +// keeps h2, broken upstream or not. +func TestUpstreamTransport_ExplicitHTTP2NeverDowngrades(t *testing.T) { + t.Setenv(EnvUpstreamHTTPVersion, string(upstreamHTTP2)) + srv := startBrokenHTTP2Server(t) + + mt := NewDirectOnly(nil) + ctx := WithSkipTLSVerify(WithDirectUpstream(context.Background())) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+srv.addr, nil) + require.NoError(t, err) + + resp, err := mt.RoundTrip(req) + if err == nil { + _ = resp.Body.Close() + } + require.Error(t, err, "NB_PROXY_UPSTREAM_HTTP_VERSION=2 must not fall back to HTTP/1.1") + assert.False(t, mt.insecure.isDowngraded(srv.addr), "an explicit version must never pin an upstream") +} + +func TestUpstreamTransport_MayDowngrade(t *testing.T) { + tests := []struct { + name string + version upstreamHTTPVersion + url string + want bool + }{ + {name: "auto over TLS", version: upstreamHTTPAuto, url: "https://backend.invalid", want: true}, + // The proxy speaks no h2c, so a cleartext upstream is already on + // HTTP/1.1 and its failures say nothing about h2. + {name: "auto cleartext", version: upstreamHTTPAuto, url: "http://backend.invalid", want: false}, + {name: "explicit 1.1", version: upstreamHTTP11, url: "https://backend.invalid", want: false}, + {name: "explicit 2", version: upstreamHTTP2, url: "https://backend.invalid", want: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + transport := newUpstreamTransport(&http.Transport{}, tc.version, nil) + req, err := http.NewRequest(http.MethodGet, tc.url, nil) + require.NoError(t, err) + + assert.Equal(t, tc.want, transport.mayDowngrade(req)) + }) + } +} + +func TestUpstreamTransport_DowngradeExpires(t *testing.T) { + transport := newUpstreamTransport(&http.Transport{}, upstreamHTTPAuto, nil) + transport.markDowngraded("backend.invalid:443") + require.True(t, transport.isDowngraded("backend.invalid:443")) + + transport.mu.Lock() + transport.downgraded["backend.invalid:443"] = time.Now().Add(-time.Second) + transport.mu.Unlock() + + assert.False(t, transport.isDowngraded("backend.invalid:443"), + "an expired pin must let the upstream be offered h2 again") + transport.mu.RLock() + _, stillTracked := transport.downgraded["backend.invalid:443"] + transport.mu.RUnlock() + assert.False(t, stillTracked, "an expired pin must not be kept around") +} + +func TestUpstreamTransport_DowngradeIsPerUpstream(t *testing.T) { + transport := newUpstreamTransport(&http.Transport{}, upstreamHTTPAuto, nil) + transport.markDowngraded("broken.invalid:443") + + assert.True(t, transport.isDowngraded("broken.invalid:443")) + assert.False(t, transport.isDowngraded("healthy.invalid:443"), + "one broken upstream must not drop the others to HTTP/1.1") +} + +func TestIsHTTP2ProtocolError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + { + name: "goaway demanding http/1.1", + err: errors.New(`http2: server sent GOAWAY and closed the connection; LastStreamID=0, ErrCode=HTTP_1_1_REQUIRED, debug=""`), + want: true, + }, + { + name: "stream error", + err: http2.StreamError{StreamID: 1, Code: http2.ErrCodeProtocol}, + want: true, + }, + { + name: "connection error", + err: http2.ConnectionError(http2.ErrCodeProtocol), + want: true, + }, + { + name: "wrapped h2 error", + err: errors.New("Get \"https://backend.invalid\": http2: client connection lost"), + want: true, + }, + // Retrying these on HTTP/1.1 fixes nothing, so they must never + // pin an upstream. + {name: "dial failure", err: errors.New("dial tcp 10.0.0.1:443: connect: connection refused"), want: false}, + {name: "tls failure", err: errors.New("tls: failed to verify certificate: x509: certificate signed by unknown authority"), want: false}, + {name: "context cancelled", err: context.Canceled, want: false}, + {name: "eof", err: io.EOF, want: false}, + {name: "nil", err: nil, want: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isHTTP2ProtocolError(tc.err)) + }) + } +} + +func TestReplayable(t *testing.T) { + t.Run("bodyless request", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "https://backend.invalid", nil) + require.NoError(t, err) + + retry, ok := replayable(req) + require.True(t, ok) + assert.Same(t, req, retry, "a bodyless request needs no clone") + }) + + t.Run("request with GetBody", func(t *testing.T) { + req, err := http.NewRequest(http.MethodPost, "https://backend.invalid", strings.NewReader("payload")) + require.NoError(t, err) + // Consume the body the way a failed RoundTrip would. + _, err = io.ReadAll(req.Body) + require.NoError(t, err) + + retry, ok := replayable(req) + require.True(t, ok) + body, err := io.ReadAll(retry.Body) + require.NoError(t, err) + assert.Equal(t, "payload", string(body), "the retry must carry a fresh copy of the body") + }) + + t.Run("streamed request", func(t *testing.T) { + req, err := http.NewRequest(http.MethodPost, "https://backend.invalid", io.NopCloser(strings.NewReader("payload"))) + require.NoError(t, err) + require.Nil(t, req.GetBody, "an opaque reader must not get a GetBody") + + _, ok := replayable(req) + assert.False(t, ok, "a body that cannot be regenerated must not be replayed") + }) +} + +// brokenHTTP2Server advertises h2 in ALPN, accepts it, and then refuses +// to serve it — the upstream behaviour that motivated the fallback. Over +// http/1.1 it answers normally, so a downgraded request succeeds. +type brokenHTTP2Server struct { + addr string + + handshakes chan struct{} +} + +func (s *brokenHTTP2Server) http2Handshakes() int { + return len(s.handshakes) +} + +func startBrokenHTTP2Server(t *testing.T) *brokenHTTP2Server { + t.Helper() + + ln, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{ + Certificates: []tls.Certificate{selfSignedCert(t)}, + NextProtos: []string{"h2", "http/1.1"}, + MinVersion: tls.VersionTLS12, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + + srv := &brokenHTTP2Server{ + addr: ln.Addr().String(), + // Buffered well past what the test drives so a stuck server + // never blocks the accept loop. + handshakes: make(chan struct{}, 64), + } + + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go srv.handle(conn) + } + }() + + return srv +} + +func (s *brokenHTTP2Server) handle(conn net.Conn) { + defer func() { _ = conn.Close() }() + + tlsConn, ok := conn.(*tls.Conn) + if !ok { + return + } + if err := tlsConn.Handshake(); err != nil { + return + } + + if tlsConn.ConnectionState().NegotiatedProtocol == "h2" { + select { + case s.handshakes <- struct{}{}: + default: + } + s.refuseHTTP2(tlsConn) + return + } + + s.serveHTTP1(tlsConn) +} + +// refuseHTTP2 completes just enough of the h2 handshake for the client +// to accept the connection, then sends the GOAWAY an upstream uses to +// say the request belongs on HTTP/1.1. +func (s *brokenHTTP2Server) refuseHTTP2(conn net.Conn) { + framer := http2.NewFramer(conn, conn) + if err := framer.WriteSettings(); err != nil { + return + } + _ = framer.WriteGoAway(0, http2.ErrCodeHTTP11Required, nil) +} + +// serveHTTP1 answers a single request with the protocol the upstream +// saw, so the test can tell which transport carried it. +func (s *brokenHTTP2Server) serveHTTP1(conn net.Conn) { + reader := bufio.NewReader(conn) + if _, err := http.ReadRequest(reader); err != nil { + return + } + + const body = "http/1.1" + _, _ = io.WriteString(conn, "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 8\r\nConnection: close\r\n\r\n"+body) +} + +func selfSignedCert(t *testing.T) tls.Certificate { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "127.0.0.1"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + require.NoError(t, err) + + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +}