diff --git a/proxy/internal/roundtrip/multi.go b/proxy/internal/roundtrip/multi.go index 567249437..1abf54a8d 100644 --- a/proxy/internal/roundtrip/multi.go +++ b/proxy/internal/roundtrip/multi.go @@ -26,8 +26,8 @@ import ( // branch at all), construct the MultiTransport via NewDirectOnly. type MultiTransport struct { embedded http.RoundTripper - direct *http.Transport - insecure *http.Transport + direct *upstreamTransport + insecure *upstreamTransport } // errNoEmbeddedTransport is returned when a request reaches the @@ -53,7 +53,6 @@ func NewMultiTransport(embedded http.RoundTripper, logger *log.Logger) *MultiTra } direct := &http.Transport{ DialContext: dialWithTimeout(dialer.DialContext), - ForceAttemptHTTP2: true, MaxIdleConns: cfg.maxIdleConns, MaxIdleConnsPerHost: cfg.maxIdleConnsPerHost, MaxConnsPerHost: cfg.maxConnsPerHost, @@ -70,8 +69,8 @@ func NewMultiTransport(embedded http.RoundTripper, logger *log.Logger) *MultiTra return &MultiTransport{ embedded: embedded, - direct: direct, - insecure: insecure, + direct: newUpstreamTransport(direct, cfg.upstreamHTTPVersion, logger), + insecure: newUpstreamTransport(insecure, cfg.upstreamHTTPVersion, logger), } } diff --git a/proxy/internal/roundtrip/multi_test.go b/proxy/internal/roundtrip/multi_test.go index 5c6cf1c97..7b8b50e81 100644 --- a/proxy/internal/roundtrip/multi_test.go +++ b/proxy/internal/roundtrip/multi_test.go @@ -5,6 +5,7 @@ import ( "io" "net/http" "net/http/httptest" + "os" "strings" "testing" "time" @@ -75,16 +76,71 @@ func TestMultiTransport_AppliesEnvOverridesToDirect(t *testing.T) { mt := NewMultiTransport(&stubRoundTripper{body: "embedded"}, nil) - assert.Equal(t, 42, mt.direct.MaxIdleConns, + assert.Equal(t, 42, mt.direct.primary.MaxIdleConns, "NB_PROXY_MAX_IDLE_CONNS must propagate to the direct transport") - assert.Equal(t, 11*time.Second, mt.direct.IdleConnTimeout, + assert.Equal(t, 11*time.Second, mt.direct.primary.IdleConnTimeout, "NB_PROXY_IDLE_CONN_TIMEOUT must propagate to the direct transport") - assert.Equal(t, 7*time.Second, mt.direct.TLSHandshakeTimeout, + assert.Equal(t, 7*time.Second, mt.direct.primary.TLSHandshakeTimeout, "NB_PROXY_TLS_HANDSHAKE_TIMEOUT must propagate to the direct transport") - assert.Equal(t, 42, mt.insecure.MaxIdleConns, + assert.Equal(t, 42, mt.insecure.primary.MaxIdleConns, "env tuning must also apply to the insecure-skip-verify direct transport") } +// TestMultiTransport_UpstreamHTTPVersion pins the protocol actually +// negotiated with an HTTPS upstream that offers both h2 and http/1.1. +// The request rides the insecure clone, so this also covers the version +// surviving http.Transport.Clone. +func TestMultiTransport_UpstreamHTTPVersion(t *testing.T) { + tests := []struct { + name string + env string + wantProto string + }{ + {name: "unset negotiates h2", env: "", wantProto: "HTTP/2.0"}, + {name: "auto negotiates h2", env: "auto", wantProto: "HTTP/2.0"}, + {name: "1.1 pins http/1.1", env: "1.1", wantProto: "HTTP/1.1"}, + {name: "2 negotiates h2", env: "2", wantProto: "HTTP/2.0"}, + {name: "unsupported value keeps the default", env: "http3", wantProto: "HTTP/2.0"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + // t.Setenv registers the restore for whatever the process + // inherited; unsetting afterwards lets the default row + // exercise a genuinely absent variable. + t.Setenv(EnvUpstreamHTTPVersion, tc.env) + if tc.env == "" { + require.NoError(t, os.Unsetenv(EnvUpstreamHTTPVersion)) + } + + // The test server's certificate isn't in any root pool, so the + // request rides the insecure branch via WithSkipTLSVerify. + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, r.Proto) + })) + srv.EnableHTTP2 = true + srv.StartTLS() + defer srv.Close() + + mt := NewDirectOnly(nil) + ctx := WithSkipTLSVerify(WithDirectUpstream(context.Background())) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, srv.URL, nil) + require.NoError(t, err) + + resp, err := mt.RoundTrip(req) + require.NoError(t, err) + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + require.NoError(t, err) + + assert.Equal(t, tc.wantProto, resp.Proto, + "client-side protocol must follow %s=%q", EnvUpstreamHTTPVersion, tc.env) + assert.Equal(t, tc.wantProto, string(body), + "the upstream must see the same protocol the client negotiated") + }) + } +} + // TestMultiTransport_NilEmbeddedErrorsWhenWGPathRequested guards // against the previous silent fallback: a MultiTransport constructed // without an embedded transport must reject requests that don't diff --git a/proxy/internal/roundtrip/netbird.go b/proxy/internal/roundtrip/netbird.go index ae3308a3e..d7b464182 100644 --- a/proxy/internal/roundtrip/netbird.go +++ b/proxy/internal/roundtrip/netbird.go @@ -82,10 +82,10 @@ type serviceNotification struct { // clientEntry holds an embedded NetBird client and tracks which services use it. type clientEntry struct { client *embed.Client - transport *http.Transport + transport *upstreamTransport // insecureTransport is a clone of transport with TLS verification disabled, // used when per-target skip_tls_verify is set. - insecureTransport *http.Transport + insecureTransport *upstreamTransport services map[ServiceKey]serviceInfo createdAt time.Time started bool @@ -414,7 +414,6 @@ func (n *NetBird) createClientEntry(ctx context.Context, accountID types.Account // not work with reverse proxied requests. transport := &http.Transport{ DialContext: dialWithTimeout(client.DialContext), - ForceAttemptHTTP2: true, MaxIdleConns: n.transportCfg.maxIdleConns, MaxIdleConnsPerHost: n.transportCfg.maxIdleConnsPerHost, MaxConnsPerHost: n.transportCfg.maxConnsPerHost, @@ -426,15 +425,14 @@ func (n *NetBird) createClientEntry(ctx context.Context, accountID types.Account ReadBufferSize: n.transportCfg.readBufferSize, DisableCompression: n.transportCfg.disableCompression, } - insecureTransport := transport.Clone() insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec return &clientEntry{ client: client, services: map[ServiceKey]serviceInfo{key: si}, - transport: transport, - insecureTransport: insecureTransport, + transport: newUpstreamTransport(transport, n.transportCfg.upstreamHTTPVersion, n.logger), + insecureTransport: newUpstreamTransport(insecureTransport, n.transportCfg.upstreamHTTPVersion, n.logger), createdAt: time.Now(), started: false, inflightMap: make(map[backendKey]chan struct{}), diff --git a/proxy/internal/roundtrip/transport.go b/proxy/internal/roundtrip/transport.go index 7c450bbb7..9e872e447 100644 --- a/proxy/internal/roundtrip/transport.go +++ b/proxy/internal/roundtrip/transport.go @@ -1,8 +1,11 @@ package roundtrip import ( + "crypto/tls" + "net/http" "os" "strconv" + "strings" "time" log "github.com/sirupsen/logrus" @@ -21,6 +24,30 @@ const ( EnvReadBufferSize = "NB_PROXY_READ_BUFFER_SIZE" EnvDisableCompression = "NB_PROXY_DISABLE_COMPRESSION" EnvMaxInflight = "NB_PROXY_MAX_INFLIGHT" + EnvUpstreamHTTPVersion = "NB_PROXY_UPSTREAM_HTTP_VERSION" +) + +// upstreamHTTPVersion selects the HTTP version the proxy uses towards an +// upstream. The explicit values are absolute: they mean the same thing +// however the transports are dialled and whatever the default becomes, +// so operator configuration survives a change of default. +type upstreamHTTPVersion string + +const ( + // upstreamHTTPAuto leaves the choice to the upstream: h2 is offered + // alongside http/1.1 in the TLS handshake and the upstream picks. + // An upstream that picks h2 and then fails to serve it is moved to + // HTTP/1.1 on its own (see upstreamTransport), which is the part + // ALPN cannot express. This is the only value whose meaning tracks + // the proxy's default. + upstreamHTTPAuto upstreamHTTPVersion = "auto" + // upstreamHTTP11 never offers h2, so the upstream sees HTTP/1.1. + upstreamHTTP11 upstreamHTTPVersion = "1.1" + // upstreamHTTP2 offers h2 in the TLS handshake and keeps it there: + // an upstream that negotiates h2 and then breaks is never moved to + // HTTP/1.1. Cleartext upstreams stay on HTTP/1.1 regardless: the + // proxy speaks no h2c. + upstreamHTTP2 upstreamHTTPVersion = "2" ) // transportConfig holds tunable parameters for the per-account HTTP transport. @@ -37,6 +64,11 @@ type transportConfig struct { disableCompression bool // maxInflight limits per-backend concurrent requests. 0 means unlimited. maxInflight int + // upstreamHTTPVersion selects the HTTP version used towards HTTPS + // upstreams. The default negotiates it with each upstream; the + // explicit values are for backends whose advertised h2 support is + // unusable and whose failure mode the negotiation cannot see. + upstreamHTTPVersion upstreamHTTPVersion } func defaultTransportConfig() transportConfig { @@ -47,6 +79,7 @@ func defaultTransportConfig() transportConfig { idleConnTimeout: 90 * time.Second, tlsHandshakeTimeout: 10 * time.Second, expectContinueTimeout: 1 * time.Second, + upstreamHTTPVersion: upstreamHTTPAuto, } } @@ -86,6 +119,9 @@ func loadTransportConfig(logger *log.Logger) transportConfig { if v, ok := envInt(EnvMaxInflight, logger); ok { cfg.maxInflight = v } + if v, ok := envUpstreamHTTPVersion(EnvUpstreamHTTPVersion, logger); ok { + cfg.upstreamHTTPVersion = v + } logger.WithFields(log.Fields{ "max_idle_conns": cfg.maxIdleConns, @@ -99,11 +135,83 @@ func loadTransportConfig(logger *log.Logger) transportConfig { "read_buffer_size": cfg.readBufferSize, "disable_compression": cfg.disableCompression, "max_inflight": cfg.maxInflight, + "upstream_http_version": cfg.upstreamHTTPVersion, }).Debug("backend transport configuration") return cfg } +// applyUpstreamHTTPVersion configures t's ALPN offer for the requested +// HTTP version. It is the single place that decides which protocols a +// transport offers, so changing the proxy's default only touches this +// function and leaves every explicit operator setting intact. What +// happens when a negotiated h2 upstream then fails belongs to +// upstreamTransport, which owns the runtime half of "auto". +// +// HTTP/1.1 is pinned by clearing ForceAttemptHTTP2 and installing an +// empty TLSNextProto, which disables h2 regardless of how the transport +// is dialled. Relying on net/http's conservative default (h2 off +// whenever a custom dialer is set) would silently start negotiating h2 +// again the day a transport switches to DialTLSContext. +func applyUpstreamHTTPVersion(t *http.Transport, version upstreamHTTPVersion) { + if version == upstreamHTTP11 { + t.ForceAttemptHTTP2 = false + t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{} + t.TLSClientConfig = withoutHTTP2ALPN(t.TLSClientConfig) + return + } + t.ForceAttemptHTTP2 = true +} + +// withoutHTTP2ALPN drops h2 from the ALPN offer. Configuring h2 makes +// net/http append h2 to the transport's TLSClientConfig, so a transport +// cloned from one that already served a request carries that offer with +// it. Left in place, the upstream would select a protocol this +// transport then refuses to speak, and the response would come back as +// h2 frames parsed as an HTTP/1.1 message. +func withoutHTTP2ALPN(cfg *tls.Config) *tls.Config { + // A nil config offers no ALPN at all, which is already HTTP/1.1. + if cfg == nil || len(cfg.NextProtos) == 0 { + return cfg + } + + protos := make([]string, 0, len(cfg.NextProtos)) + for _, proto := range cfg.NextProtos { + if proto == "h2" { + continue + } + protos = append(protos, proto) + } + if len(protos) == len(cfg.NextProtos) { + return cfg + } + + // Clone rather than edit in place: the caller may share this config + // with the transport it was cloned from. + stripped := cfg.Clone() + stripped.NextProtos = protos + + return stripped +} + +// envUpstreamHTTPVersion reads an upstream HTTP version from the +// environment. An unrecognised value warns and leaves the default in +// place rather than guessing at the operator's intent. +func envUpstreamHTTPVersion(key string, logger *log.Logger) (upstreamHTTPVersion, bool) { + s := strings.TrimSpace(os.Getenv(key)) + if s == "" { + return "", false + } + switch v := upstreamHTTPVersion(strings.ToLower(s)); v { + case upstreamHTTPAuto, upstreamHTTP11, upstreamHTTP2: + return v, true + default: + logger.Warnf("ignoring unsupported %s=%q, expected one of %q, %q, %q", + key, s, upstreamHTTPAuto, upstreamHTTP11, upstreamHTTP2) + return "", false + } +} + func envInt(key string, logger *log.Logger) (int, bool) { s := os.Getenv(key) if s == "" { diff --git a/proxy/internal/roundtrip/upstream.go b/proxy/internal/roundtrip/upstream.go new file mode 100644 index 000000000..6602c836e --- /dev/null +++ b/proxy/internal/roundtrip/upstream.go @@ -0,0 +1,455 @@ +package roundtrip + +import ( + "errors" + "net" + "net/http" + "net/netip" + "net/url" + "strings" + "sync" + "time" + + log "github.com/sirupsen/logrus" +) + +// upstreamDowngradeTTL is how long an upstream stays pinned to HTTP/1.1 +// after an h2 failure that only implied it cannot serve h2. Bounded so a +// fixed or replaced backend returns to h2 without restarting the proxy. +// A pin the upstream asked for itself does not expire — see downgrade. +const upstreamDowngradeTTL = 10 * time.Minute + +// downgrade is an upstream's HTTP/1.1 pin. +type downgrade struct { + // expiry is when the pin lapses and the upstream is offered h2 + // again. The zero time means it never does: the upstream answered + // HTTP_1_1_REQUIRED, which is a statement about how it is + // configured, not a fault that may clear on its own. Re-probing + // that every upstreamDowngradeTTL would buy nothing but a failed + // request per interval, so the pin holds until the transport goes + // away with the proxy or the account's client. + expiry time.Time +} + +// permanent reports whether the upstream asked for this pin itself. +func (d downgrade) permanent() bool { + return d.expiry.IsZero() +} + +// active reports whether the pin still stands at now. +func (d downgrade) active(now time.Time) bool { + return d.permanent() || now.Before(d.expiry) +} + +// upstreamTransport carries requests to a single upstream family (one +// TLS configuration) and implements what upstreamHTTPAuto means. +// +// ALPN already lets the upstream pick the protocol: primary offers both +// h2 and http/1.1 and the server chooses. What ALPN cannot express is +// an upstream that selects h2 and then fails to speak it — the case +// this type handles. The first h2-level failure for a host pins that +// host to fallback, an HTTP/1.1-only clone of primary, and the request +// is retried there when it can be replayed. +// +// The downgrade is per upstream host, not per transport: one broken +// backend must not drop every other backend to HTTP/1.1. +type upstreamTransport struct { + // primary is the configured transport: h2 offered in ALPN for + // upstreamHTTPAuto and upstreamHTTP2, HTTP/1.1-only for + // upstreamHTTP11. + primary *http.Transport + // version decides whether a downgrade may happen at all. Only + // upstreamHTTPAuto downgrades; the explicit values are absolute. + version upstreamHTTPVersion + logger *log.Logger + + // fallbackMu guards the lazy fallback clone: most deployments never + // hit a broken h2 upstream and should not pay for a second + // connection pool. + fallbackMu sync.Mutex + fallback *http.Transport + + mu sync.RWMutex + // downgraded maps an upstream host to its HTTP/1.1 pin. + downgraded map[string]downgrade +} + +// newUpstreamTransport wraps base for the requested HTTP version. base +// must not be used directly afterwards: the wrapper owns it, including +// its connection pool. +func newUpstreamTransport(base *http.Transport, version upstreamHTTPVersion, logger *log.Logger) *upstreamTransport { + if logger == nil { + logger = log.StandardLogger() + } + applyUpstreamHTTPVersion(base, version) + + return &upstreamTransport{ + primary: base, + version: version, + logger: logger, + downgraded: make(map[string]downgrade), + } +} + +// RoundTrip implements http.RoundTripper. +func (t *upstreamTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if !t.mayDowngrade(req) { + return t.primary.RoundTrip(req) + } + + host := upstreamKey(req.URL) + if t.isDowngraded(host) { + return t.http1().RoundTrip(req) + } + + resp, err := t.primary.RoundTrip(req) + if err == nil || !isHTTP2ProtocolError(err) { + return resp, err + } + + // HTTP_1_1_REQUIRED is the upstream saying it will not serve this + // request over h2 however often it is asked — IIS answers it for + // Windows Authentication and for client-certificate sites, where + // the cause is site configuration rather than a passing fault. + t.markDowngraded(host, isHTTP11Required(err)) + + if !safeToRetry(req, err) { + // The upstream may have carried out the request before failing + // to answer over h2, and repeating it could duplicate whatever + // it did. The host is pinned either way, so the next request + // goes out over HTTP/1.1. + t.logger.WithFields(log.Fields{ + "upstream": host, + "method": req.Method, + }).Debug("not retrying over HTTP/1.1: the upstream may already have applied this request") + return nil, err + } + + retry, ok := replayable(req) + if !ok { + // The body is already consumed and cannot be regenerated, so + // this request fails, and the pin carries the next one. + return nil, err + } + return t.http1().RoundTrip(retry) +} + +// CloseIdleConnections closes idle connections on both pools. +func (t *upstreamTransport) CloseIdleConnections() { + t.primary.CloseIdleConnections() + if fallback := t.existingHTTP1(); fallback != nil { + fallback.CloseIdleConnections() + } +} + +// mayDowngrade reports whether a failed request is a downgrade +// candidate. Only upstreamHTTPAuto downgrades, and only for TLS +// upstreams: the proxy speaks no h2c, so a cleartext upstream is +// already on HTTP/1.1 and an error there says nothing about h2. +func (t *upstreamTransport) mayDowngrade(req *http.Request) bool { + return t.version == upstreamHTTPAuto && req.URL != nil && req.URL.Scheme == "https" +} + +func (t *upstreamTransport) isDowngraded(host string) bool { + t.mu.RLock() + pin, ok := t.downgraded[host] + t.mu.RUnlock() + + if !ok { + return false + } + if pin.active(time.Now()) { + return true + } + + t.mu.Lock() + defer t.mu.Unlock() + + // Re-read under the write lock rather than trusting the expired pin + // from above: a concurrent request may have re-pinned the host since, + // and that pin decides this request too. Reporting the stale read + // would send one request back to h2 against a live pin. + pin, ok = t.downgraded[host] + if !ok { + return false + } + if pin.active(time.Now()) { + return true + } + delete(t.downgraded, host) + + return false +} + +// markDowngraded pins host to HTTP/1.1. permanent marks a pin the +// upstream asked for; anything else lapses after upstreamDowngradeTTL so +// a repaired backend is offered h2 again. +func (t *upstreamTransport) markDowngraded(host string, permanent bool) { + now := time.Now() + pin := downgrade{expiry: now.Add(upstreamDowngradeTTL)} + if permanent { + pin = downgrade{} + } + + t.mu.Lock() + previous, pinned := t.downgraded[host] + // A permanent pin is never weakened back into an expiring one: the + // upstream has already said h2 is not on offer. + promoted := pinned && !previous.permanent() && permanent + if !pinned || !previous.permanent() { + t.downgraded[host] = pin + } + for h, existing := range t.downgraded { + if !existing.active(now) { + delete(t.downgraded, h) + } + } + t.mu.Unlock() + + // Log a new pin, and a pin the upstream has since asked to make + // permanent — otherwise an operator would only ever see the "for the + // next 10m" line and never learn the upstream settled the question. + if pinned && !promoted { + return + } + + entry := t.logger.WithField("upstream", host) + if permanent { + entry.Warnf("upstream answered HTTP_1_1_REQUIRED, using HTTP/1.1 for it from now on") + return + } + entry.Warnf("upstream negotiated HTTP/2 but failed to serve it, using HTTP/1.1 for the next %s (set %s=1.1 to pin it)", + upstreamDowngradeTTL, EnvUpstreamHTTPVersion) +} + +// http1 returns the HTTP/1.1-only clone, creating it on first use. +func (t *upstreamTransport) http1() *http.Transport { + t.fallbackMu.Lock() + defer t.fallbackMu.Unlock() + + if t.fallback == nil { + fallback := t.primary.Clone() + applyUpstreamHTTPVersion(fallback, upstreamHTTP11) + t.fallback = fallback + } + + return t.fallback +} + +// existingHTTP1 returns the fallback transport only if it was already +// created, so housekeeping never allocates a second connection pool for +// an upstream that never needed one. +func (t *upstreamTransport) existingHTTP1() *http.Transport { + t.fallbackMu.Lock() + defer t.fallbackMu.Unlock() + + return t.fallback +} + +// upstreamKey normalizes an authority for use as a pin key, so one +// upstream cannot end up with two independent pins. DNS labels compare +// case-insensitively, and the default HTTPS port is implied — every +// downgrade path is TLS-only, so a bare host and the same host on :443 +// are the same upstream. +func upstreamKey(u *url.URL) string { + host := normalizeUpstreamHost(u.Hostname()) + + port := u.Port() + if port == "" || port == "443" { + return host + } + + // JoinHostPort rather than concatenation: an IPv6 literal needs its + // brackets back after Hostname stripped them. + return net.JoinHostPort(host, port) +} + +// normalizeUpstreamHost folds the spellings of one host onto a single +// key. An IP literal goes through netip so that the several textual +// forms of one address (case, leading zeroes, a compressed run) collapse +// and a v4-mapped address keys as the v4 address it is. A zone +// identifier is left exactly as written: it names an interface, and +// interface names are case-sensitive on the systems that have them, so +// %eth0 and %ETH0 may be different links and must not share a pin. +// Anything that is not an IP literal is a DNS name, which compares +// case-insensitively. +func normalizeUpstreamHost(host string) string { + if addr, err := netip.ParseAddr(host); err == nil { + return addr.Unmap().String() + } + + return strings.ToLower(host) +} + +// safeToRetry reports whether req may be sent a second time over +// HTTP/1.1 after err ended its h2 attempt. +// +// A failure at the h2 layer does not say whether the upstream already +// carried out the request, so replaying one that changes state could +// duplicate it. Two cases are safe: a request whose repetition is +// harmless by definition, and an upstream that told us it processed +// nothing on the connection. The second is what makes the IIS case work +// for every method — a site requiring HTTP/1.1 refuses at stream 0, +// before the request is looked at. +func safeToRetry(req *http.Request, err error) bool { + return idempotent(req) || upstreamProcessedNothing(err) +} + +// idempotent reports whether repeating req is defined to be harmless. +// It mirrors net/http's own retry rule (Request.isReplayable): a method +// with no side effects, or a caller that promised the upstream +// deduplicates by key. +func idempotent(req *http.Request) bool { + if req.Header.Get("Idempotency-Key") != "" || req.Header.Get("X-Idempotency-Key") != "" { + return true + } + + switch req.Method { + // An empty method means GET, as in net/http. + case "", http.MethodGet, http.MethodHead, http.MethodOptions, http.MethodTrace: + return true + } + + return false +} + +// upstreamProcessedNothing reports whether err describes a GOAWAY that +// named this request's stream as one the upstream had not received, so +// it cannot have acted on it. A stream error says the opposite: the +// stream was open, so the request had been delivered. +func upstreamProcessedNothing(err error) bool { + if err == nil { + return false + } + + msg := transportError(err).Error() + + return strings.Contains(msg, goAwayStreamNotReceivedMarker) || + strings.Contains(msg, goAwayNothingProcessedMarker) +} + +// replayable returns a request that can be sent a second time, or +// ok=false when the body is gone. A RoundTripper consumes and closes +// the body it was given, so a retry needs either no body at all or +// GetBody to produce a fresh one. +func replayable(req *http.Request) (*http.Request, bool) { + if req.Body == nil || req.Body == http.NoBody { + return req, true + } + if req.GetBody == nil { + return nil, false + } + + body, err := req.GetBody() + if err != nil { + return nil, false + } + + retry := req.Clone(req.Context()) + retry.Body = body + + return retry, true +} + +// http2ErrorMarkers are the substrings that identify an HTTP/2 protocol +// failure. net/http bundles its own private copy of the http2 package, +// so its errors cannot be matched by type from here: http2.StreamError +// and friends in x/net are different types from the ones a +// bundled-h2 transport returns. The strings below are the formats those +// bundled errors print, and they are specific to h2 framing — a +// downgrade must never be triggered by an ordinary network or TLS +// error, which retrying on HTTP/1.1 would not fix. +var http2ErrorMarkers = []string{ + // Transport-level h2 failures, e.g. + // "http2: server sent GOAWAY and closed the connection". + "http2:", + // http2.StreamError, e.g. "stream error: stream ID 1; PROTOCOL_ERROR". + "stream error: stream ID", + // http2.ConnectionError, e.g. "connection error: PROTOCOL_ERROR". + "connection error: ", + // The code an upstream sends to say the request must be retried + // over HTTP/1.1, as a GOAWAY or on the stream. + http11RequiredMarker, +} + +const ( + // http11RequiredMarker is the error code an upstream sends to say the + // request belongs on HTTP/1.1. Unlike the other markers it is not a + // fault: the upstream is describing its own configuration. + http11RequiredMarker = "HTTP_1_1_REQUIRED" + + // A GOAWAY carrying NO_ERROR closes a connection without complaint: + // a server draining before shutdown, recycling an application pool, + // capping requests per connection. The upstream speaks h2 perfectly + // well, so this must never pin it. The two spellings are the two + // formats the bundled transport prints the code in. + goAwayNoErrorEqualsMarker = "ErrCode=NO_ERROR" + goAwayNoErrorColonMarker = "ErrCode:NO_ERROR" + // gracefulGoAwayMarker is errClientConnGotGoAway, which the bundled + // transport raises for a stream the server never received on a + // connection it is shutting down gracefully. It normally retries + // those itself on a new connection and this never surfaces. + gracefulGoAwayMarker = "Transport received Server's graceful shutdown GOAWAY" + + // goAwayStreamNotReceivedMarker is the bundled transport's abort for + // the first stream on a connection whose GOAWAY carried a real error + // code — the IIS case. It sits in the same "streamID > LastStreamID" + // branch as the graceful abort, so the server had not received the + // stream (see net/http's h2_bundle.go). + goAwayStreamNotReceivedMarker = "Transport received GOAWAY from server ErrCode:" + // goAwayNothingProcessedMarker is a GoAwayError naming stream 0 as + // the last one received, which says the same thing. The trailing + // comma keeps it from matching LastStreamID=10 and the rest. + goAwayNothingProcessedMarker = "LastStreamID=0," +) + +// isHTTP11Required reports whether the upstream itself asked for +// HTTP/1.1, rather than merely failing at h2. +func isHTTP11Required(err error) bool { + return err != nil && strings.Contains(transportError(err).Error(), http11RequiredMarker) +} + +// isHTTP2ProtocolError reports whether err says the upstream cannot +// serve the h2 it negotiated. +func isHTTP2ProtocolError(err error) bool { + if err == nil { + return false + } + + msg := transportError(err).Error() + + // A graceful GOAWAY is routine connection management, not an + // upstream that cannot serve h2. + if isGracefulGoAway(msg) { + return false + } + + for _, marker := range http2ErrorMarkers { + if strings.Contains(msg, marker) { + return true + } + } + + return false +} + +// isGracefulGoAway reports whether msg describes a GOAWAY sent to close +// a healthy connection rather than to report an inability to serve h2. +func isGracefulGoAway(msg string) bool { + return strings.Contains(msg, goAwayNoErrorEqualsMarker) || + strings.Contains(msg, goAwayNoErrorColonMarker) || + strings.Contains(msg, gracefulGoAwayMarker) +} + +// transportError strips a *url.Error wrapper, which prefixes the request +// URL to the message. Markers are matched as substrings, so a URL left +// in place could classify an ordinary dial or TLS failure as an h2 one +// on the strength of the path alone. +func transportError(err error) error { + var urlErr *url.Error + if errors.As(err, &urlErr) && urlErr.Err != nil { + return urlErr.Err + } + + return err +} diff --git a/proxy/internal/roundtrip/upstream_test.go b/proxy/internal/roundtrip/upstream_test.go new file mode 100644 index 000000000..a83570ef3 --- /dev/null +++ b/proxy/internal/roundtrip/upstream_test.go @@ -0,0 +1,587 @@ +package roundtrip + +import ( + "bufio" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "fmt" + "io" + "math/big" + "net" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/http2" +) + +// TestUpstreamTransport_AutoFallsBackOnBrokenHTTP2 covers the case ALPN +// cannot express: the upstream advertises h2, picks it, and then cannot +// serve it. The request must still succeed, over HTTP/1.1, and the +// upstream must stay on HTTP/1.1 for the requests that follow. +func TestUpstreamTransport_AutoFallsBackOnBrokenHTTP2(t *testing.T) { + t.Setenv(EnvUpstreamHTTPVersion, string(upstreamHTTPAuto)) + srv := startBrokenHTTP2Server(t) + + mt := NewDirectOnly(nil) + ctx := WithSkipTLSVerify(WithDirectUpstream(context.Background())) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+srv.addr, nil) + require.NoError(t, err) + + resp, err := mt.RoundTrip(req) + require.NoError(t, err, "a replayable request must be retried on HTTP/1.1 instead of failing") + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + require.NoError(t, err) + assert.Equal(t, "HTTP/1.1", resp.Proto, "the retry must ride the HTTP/1.1 transport") + assert.Equal(t, "http/1.1", string(body), "the upstream must see an http/1.1 ALPN offer on the retry") + + assert.True(t, mt.insecure.isDowngraded(srv.addr), + "the upstream must stay pinned to HTTP/1.1 after proving it cannot serve h2") + mt.insecure.mu.RLock() + pin := mt.insecure.downgraded[srv.addr] + mt.insecure.mu.RUnlock() + assert.True(t, pin.permanent(), + "an upstream answering HTTP_1_1_REQUIRED must not be re-probed for h2") + + // The second request must not repeat the h2 attempt: the server + // counts h2 handshakes, so a repeat would show up here. + h2Attempts := srv.http2Handshakes() + req, err = http.NewRequestWithContext(ctx, http.MethodGet, "https://"+srv.addr, nil) + require.NoError(t, err) + resp, err = mt.RoundTrip(req) + require.NoError(t, err) + _ = resp.Body.Close() + + assert.Equal(t, "HTTP/1.1", resp.Proto, "a pinned upstream must go straight to HTTP/1.1") + assert.Equal(t, h2Attempts, srv.http2Handshakes(), + "a pinned upstream must not be probed for h2 again until the pin expires") +} + +// TestUpstreamTransport_ExplicitHTTP2NeverDowngrades pins the promise +// that the explicit values are absolute: an operator who asked for h2 +// keeps h2, broken upstream or not. +func TestUpstreamTransport_ExplicitHTTP2NeverDowngrades(t *testing.T) { + t.Setenv(EnvUpstreamHTTPVersion, string(upstreamHTTP2)) + srv := startBrokenHTTP2Server(t) + + mt := NewDirectOnly(nil) + ctx := WithSkipTLSVerify(WithDirectUpstream(context.Background())) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+srv.addr, nil) + require.NoError(t, err) + + resp, err := mt.RoundTrip(req) + if err == nil { + _ = resp.Body.Close() + } + require.Error(t, err, "NB_PROXY_UPSTREAM_HTTP_VERSION=2 must not fall back to HTTP/1.1") + assert.False(t, mt.insecure.isDowngraded(srv.addr), "an explicit version must never pin an upstream") +} + +// TestUpstreamTransport_AutoDoesNotReplayUnsafeRequests covers the +// other half of the fallback: an h2 failure says nothing about whether +// the upstream already applied the request, so a state-changing one is +// not replayed. The host is still pinned, so the next request rides +// HTTP/1.1 without a second h2 attempt. +func TestUpstreamTransport_AutoDoesNotReplayUnsafeRequests(t *testing.T) { + t.Setenv(EnvUpstreamHTTPVersion, string(upstreamHTTPAuto)) + srv := startBrokenHTTP2Server(t) + // A stream error means the stream was open, so the upstream had the + // request in hand — unlike the GOAWAY at stream 0 the fake server + // sends, which states it processed nothing. + streamErr := http2.StreamError{StreamID: 1, Code: http2.ErrCodeProtocol} + + mt := NewDirectOnly(nil) + transport := mt.insecure + ctx := WithSkipTLSVerify(WithDirectUpstream(context.Background())) + + assert.False(t, safeToRetry(newTestRequest(t, ctx, http.MethodPost, srv.addr), streamErr), + "a POST must not be replayed after a failure that may have been applied") + assert.True(t, safeToRetry(newTestRequest(t, ctx, http.MethodGet, srv.addr), streamErr), + "a GET is safe to replay whatever the failure was") + + // The fake server's GOAWAY names stream 0, so even a POST is safe + // there and the request must succeed over HTTP/1.1. + resp, err := transport.RoundTrip(newTestRequest(t, ctx, http.MethodPost, srv.addr)) + require.NoError(t, err, "a GOAWAY at stream 0 means the upstream applied nothing, so the POST may be replayed") + body, err := io.ReadAll(resp.Body) + _ = resp.Body.Close() + require.NoError(t, err) + assert.Equal(t, "http/1.1", string(body), "the retry must reach the upstream over http/1.1") + + h2Attempts := srv.http2Handshakes() + resp, err = transport.RoundTrip(newTestRequest(t, ctx, http.MethodPost, srv.addr)) + require.NoError(t, err) + _ = resp.Body.Close() + assert.Equal(t, h2Attempts, srv.http2Handshakes(), + "the pin must carry later requests without another h2 attempt") +} + +func newTestRequest(t *testing.T, ctx context.Context, method, addr string) *http.Request { + t.Helper() + + req, err := http.NewRequestWithContext(ctx, method, "https://"+addr, strings.NewReader("payload")) + require.NoError(t, err) + + return req +} + +func TestUpstreamKey(t *testing.T) { + tests := []struct { + name string + url string + want string + }{ + {name: "host", url: "https://backend.invalid/path", want: "backend.invalid"}, + // DNS is case-insensitive, so these are one upstream and must + // share one pin. + {name: "mixed case", url: "https://Backend.INVALID/path", want: "backend.invalid"}, + // The default port is implied on every path that can downgrade. + {name: "explicit default port", url: "https://backend.invalid:443/", want: "backend.invalid"}, + {name: "non-default port", url: "https://backend.invalid:8443/", want: "backend.invalid:8443"}, + // An IPv6 literal needs its brackets back after Hostname strips + // them, or the key is not a dialable authority. + {name: "ipv6 default port", url: "https://[2001:db8::1]/", want: "2001:db8::1"}, + {name: "ipv6 with port", url: "https://[2001:db8::1]:8443/", want: "[2001:db8::1]:8443"}, + // One address in three spellings: hex case, a leading zero and an + // uncompressed zero run are all the same upstream. + {name: "ipv6 upper case", url: "https://[2001:DB8::1]/", want: "2001:db8::1"}, + {name: "ipv6 leading zero", url: "https://[2001:0db8::1]/", want: "2001:db8::1"}, + {name: "ipv6 uncompressed", url: "https://[2001:db8:0:0:0:0:0:1]/", want: "2001:db8::1"}, + // A v4-mapped address is the v4 address, not a second upstream. + {name: "v4-mapped", url: "https://[::ffff:192.0.2.1]/", want: "192.0.2.1"}, + // A zone names an interface, and interface names are + // case-sensitive, so these two are different links. + {name: "ipv6 zone", url: "https://[fe80::1%25eth0]/", want: "fe80::1%eth0"}, + {name: "ipv6 zone upper case", url: "https://[fe80::1%25ETH0]/", want: "fe80::1%ETH0"}, + // The address before the zone still normalizes. + {name: "ipv6 zone with upper-case address", url: "https://[FE80::1%25eth0]/", want: "fe80::1%eth0"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + parsed, err := url.Parse(tc.url) + require.NoError(t, err) + + assert.Equal(t, tc.want, upstreamKey(parsed)) + }) + } +} + +func TestSafeToRetry(t *testing.T) { + streamErr := http2.StreamError{StreamID: 1, Code: http2.ErrCodeProtocol} + goAwayAtZero := errors.New(`http2: server sent GOAWAY and closed the connection; LastStreamID=0, ErrCode=HTTP_1_1_REQUIRED, debug=""`) + goAwayLater := errors.New(`http2: server sent GOAWAY and closed the connection; LastStreamID=11, ErrCode=PROTOCOL_ERROR, debug=""`) + + tests := []struct { + name string + method string + headers map[string]string + err error + want bool + }{ + {name: "get", method: http.MethodGet, err: streamErr, want: true}, + {name: "head", method: http.MethodHead, err: streamErr, want: true}, + {name: "options", method: http.MethodOptions, err: streamErr, want: true}, + {name: "trace", method: http.MethodTrace, err: streamErr, want: true}, + {name: "post", method: http.MethodPost, err: streamErr, want: false}, + {name: "put", method: http.MethodPut, err: streamErr, want: false}, + {name: "patch", method: http.MethodPatch, err: streamErr, want: false}, + {name: "delete", method: http.MethodDelete, err: streamErr, want: false}, + // The upstream reported it handled nothing, so repeating the + // request cannot duplicate anything. + {name: "post with goaway at stream 0", method: http.MethodPost, err: goAwayAtZero, want: true}, + // What the bundled transport actually raises for the first + // stream on a connection the upstream GOAWAYs with a real error + // code, which is the shape a real IIS site produces. + { + name: "post with first-stream goaway abort", + method: http.MethodPost, + err: errors.New("http2: Transport received GOAWAY from server ErrCode:HTTP_1_1_REQUIRED"), + want: true, + }, + // It handled earlier streams, so this one may have been applied. + {name: "post with goaway after other streams", method: http.MethodPost, err: goAwayLater, want: false}, + { + name: "post with idempotency key", + method: http.MethodPost, + headers: map[string]string{"Idempotency-Key": "abc"}, + err: streamErr, + want: true, + }, + { + name: "post with prefixed idempotency key", + method: http.MethodPost, + headers: map[string]string{"X-Idempotency-Key": "abc"}, + err: streamErr, + want: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + req, err := http.NewRequest(tc.method, "https://backend.invalid", nil) + require.NoError(t, err) + for k, v := range tc.headers { + req.Header.Set(k, v) + } + + assert.Equal(t, tc.want, safeToRetry(req, tc.err)) + }) + } +} + +func TestUpstreamTransport_MayDowngrade(t *testing.T) { + tests := []struct { + name string + version upstreamHTTPVersion + url string + want bool + }{ + {name: "auto over TLS", version: upstreamHTTPAuto, url: "https://backend.invalid", want: true}, + // The proxy speaks no h2c, so a cleartext upstream is already on + // HTTP/1.1 and its failures say nothing about h2. + {name: "auto cleartext", version: upstreamHTTPAuto, url: "http://backend.invalid", want: false}, + {name: "explicit 1.1", version: upstreamHTTP11, url: "https://backend.invalid", want: false}, + {name: "explicit 2", version: upstreamHTTP2, url: "https://backend.invalid", want: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + transport := newUpstreamTransport(&http.Transport{}, tc.version, nil) + req, err := http.NewRequest(http.MethodGet, tc.url, nil) + require.NoError(t, err) + + assert.Equal(t, tc.want, transport.mayDowngrade(req)) + }) + } +} + +func TestUpstreamTransport_DowngradeExpires(t *testing.T) { + transport := newUpstreamTransport(&http.Transport{}, upstreamHTTPAuto, nil) + transport.markDowngraded("backend.invalid:443", false) + require.True(t, transport.isDowngraded("backend.invalid:443")) + + transport.mu.Lock() + transport.downgraded["backend.invalid:443"] = downgrade{expiry: time.Now().Add(-time.Second)} + transport.mu.Unlock() + + assert.False(t, transport.isDowngraded("backend.invalid:443"), + "an expired pin must let the upstream be offered h2 again") + transport.mu.RLock() + _, stillTracked := transport.downgraded["backend.invalid:443"] + transport.mu.RUnlock() + assert.False(t, stillTracked, "an expired pin must not be kept around") +} + +func TestUpstreamTransport_DowngradeIsPerUpstream(t *testing.T) { + transport := newUpstreamTransport(&http.Transport{}, upstreamHTTPAuto, nil) + transport.markDowngraded("broken.invalid:443", false) + + assert.True(t, transport.isDowngraded("broken.invalid:443")) + assert.False(t, transport.isDowngraded("healthy.invalid:443"), + "one broken upstream must not drop the others to HTTP/1.1") +} + +// TestUpstreamTransport_HTTP11RequiredPinIsPermanent covers the IIS +// case: HTTP_1_1_REQUIRED describes how the upstream is configured +// (Windows Authentication, client certificates), so re-probing it every +// upstreamDowngradeTTL would only buy a failed request per interval. +func TestUpstreamTransport_HTTP11RequiredPinIsPermanent(t *testing.T) { + transport := newUpstreamTransport(&http.Transport{}, upstreamHTTPAuto, nil) + transport.markDowngraded("iis.invalid:443", true) + + transport.mu.RLock() + pin := transport.downgraded["iis.invalid:443"] + transport.mu.RUnlock() + + assert.True(t, pin.permanent(), "an upstream that asked for HTTP/1.1 must not be re-probed") + assert.True(t, pin.active(time.Now().Add(100*upstreamDowngradeTTL)), + "a permanent pin must outlive any TTL") +} + +func TestUpstreamTransport_PermanentPinSurvivesLaterFailures(t *testing.T) { + transport := newUpstreamTransport(&http.Transport{}, upstreamHTTPAuto, nil) + transport.markDowngraded("iis.invalid:443", true) + // A later ambiguous failure for the same upstream must not turn the + // permanent pin into an expiring one. + transport.markDowngraded("iis.invalid:443", false) + + transport.mu.RLock() + pin := transport.downgraded["iis.invalid:443"] + transport.mu.RUnlock() + + assert.True(t, pin.permanent(), "a permanent pin must never be weakened") +} + +func TestIsHTTP11Required(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + { + name: "goaway", + err: errors.New(`http2: server sent GOAWAY and closed the connection; LastStreamID=0, ErrCode=HTTP_1_1_REQUIRED, debug=""`), + want: true, + }, + { + name: "stream error", + err: http2.StreamError{StreamID: 1, Code: http2.ErrCodeHTTP11Required}, + want: true, + }, + { + name: "other h2 failure", + err: http2.StreamError{StreamID: 1, Code: http2.ErrCodeProtocol}, + want: false, + }, + {name: "nil", err: nil, want: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isHTTP11Required(tc.err)) + }) + } +} + +func TestIsHTTP2ProtocolError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + { + name: "goaway demanding http/1.1", + err: errors.New(`http2: server sent GOAWAY and closed the connection; LastStreamID=0, ErrCode=HTTP_1_1_REQUIRED, debug=""`), + want: true, + }, + { + name: "stream error", + err: http2.StreamError{StreamID: 1, Code: http2.ErrCodeProtocol}, + want: true, + }, + { + name: "connection error", + err: http2.ConnectionError(http2.ErrCodeProtocol), + want: true, + }, + { + name: "wrapped h2 error", + err: errors.New("Get \"https://backend.invalid\": http2: client connection lost"), + want: true, + }, + // A GOAWAY with NO_ERROR is a server draining a connection — + // recycling an application pool, capping requests per + // connection, shutting down gracefully. It speaks h2 fine. + { + name: "graceful goaway", + err: errors.New(`http2: server sent GOAWAY and closed the connection; LastStreamID=9, ErrCode=NO_ERROR, debug=""`), + want: false, + }, + { + name: "graceful shutdown abort", + err: errors.New("http2: Transport received Server's graceful shutdown GOAWAY"), + want: false, + }, + // Markers are substrings, so a URL carried by a *url.Error must + // not be able to classify a plain failure as an h2 one. + { + name: "url error whose path looks like a marker", + err: &url.Error{ + Op: "Get", + URL: "https://backend.invalid/http2:/connection error: x", + Err: errors.New("dial tcp 10.0.0.1:443: connect: connection refused"), + }, + want: false, + }, + // Retrying these on HTTP/1.1 fixes nothing, so they must never + // pin an upstream. + {name: "dial failure", err: errors.New("dial tcp 10.0.0.1:443: connect: connection refused"), want: false}, + {name: "tls failure", err: errors.New("tls: failed to verify certificate: x509: certificate signed by unknown authority"), want: false}, + {name: "context cancelled", err: context.Canceled, want: false}, + {name: "eof", err: io.EOF, want: false}, + {name: "nil", err: nil, want: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isHTTP2ProtocolError(tc.err)) + }) + } +} + +func TestReplayable(t *testing.T) { + t.Run("bodyless request", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "https://backend.invalid", nil) + require.NoError(t, err) + + retry, ok := replayable(req) + require.True(t, ok) + assert.Same(t, req, retry, "a bodyless request needs no clone") + }) + + t.Run("request with GetBody", func(t *testing.T) { + req, err := http.NewRequest(http.MethodPost, "https://backend.invalid", strings.NewReader("payload")) + require.NoError(t, err) + // Consume the body the way a failed RoundTrip would. + _, err = io.ReadAll(req.Body) + require.NoError(t, err) + + retry, ok := replayable(req) + require.True(t, ok) + body, err := io.ReadAll(retry.Body) + require.NoError(t, err) + assert.Equal(t, "payload", string(body), "the retry must carry a fresh copy of the body") + }) + + t.Run("streamed request", func(t *testing.T) { + req, err := http.NewRequest(http.MethodPost, "https://backend.invalid", io.NopCloser(strings.NewReader("payload"))) + require.NoError(t, err) + require.Nil(t, req.GetBody, "an opaque reader must not get a GetBody") + + _, ok := replayable(req) + assert.False(t, ok, "a body that cannot be regenerated must not be replayed") + }) +} + +// brokenHTTP2Server advertises h2 in ALPN, accepts it, and then refuses +// to serve it — the upstream behaviour that motivated the fallback. Over +// http/1.1 it answers normally, so a downgraded request succeeds. +type brokenHTTP2Server struct { + addr string + + handshakes chan struct{} +} + +func (s *brokenHTTP2Server) http2Handshakes() int { + return len(s.handshakes) +} + +func startBrokenHTTP2Server(t *testing.T) *brokenHTTP2Server { + t.Helper() + + ln, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{ + Certificates: []tls.Certificate{selfSignedCert(t)}, + NextProtos: []string{"h2", "http/1.1"}, + MinVersion: tls.VersionTLS12, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = ln.Close() }) + + srv := &brokenHTTP2Server{ + addr: ln.Addr().String(), + // Buffered well past what the test drives so a stuck server + // never blocks the accept loop. + handshakes: make(chan struct{}, 64), + } + + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go srv.handle(conn) + } + }() + + return srv +} + +func (s *brokenHTTP2Server) handle(conn net.Conn) { + defer func() { _ = conn.Close() }() + + tlsConn, ok := conn.(*tls.Conn) + if !ok { + return + } + if err := tlsConn.Handshake(); err != nil { + return + } + + proto := tlsConn.ConnectionState().NegotiatedProtocol + if proto == "h2" { + select { + case s.handshakes <- struct{}{}: + default: + } + s.refuseHTTP2(tlsConn) + return + } + + s.serveHTTP1(tlsConn, proto) +} + +// refuseHTTP2 completes just enough of the h2 handshake for the client +// to accept the connection, then sends the GOAWAY an upstream uses to +// say the request belongs on HTTP/1.1. +// +// The client is still writing its preface and request while the GOAWAY +// goes out, so the connection is drained before the caller closes it. +// Closing a socket with unread bytes still in its receive buffer makes +// the kernel answer with RST, which reaches the client as a write error +// rather than the GOAWAY — no h2 error, so no downgrade, and the test +// fails on the error the client saw first. +func (s *brokenHTTP2Server) refuseHTTP2(conn net.Conn) { + framer := http2.NewFramer(conn, conn) + if err := framer.WriteSettings(); err != nil { + return + } + if err := framer.WriteGoAway(0, http2.ErrCodeHTTP11Required, nil); err != nil { + return + } + + // The client closes its side once it has read the GOAWAY, which ends + // the drain; the deadline is only a backstop against a client that + // never does. + _ = conn.SetReadDeadline(time.Now().Add(10 * time.Second)) + _, _ = io.Copy(io.Discard, conn) +} + +// serveHTTP1 answers a single request with the ALPN protocol the +// upstream actually settled on, so a test asserting on the body is +// checking what the upstream saw rather than a constant. +func (s *brokenHTTP2Server) serveHTTP1(conn net.Conn, alpn string) { + reader := bufio.NewReader(conn) + if _, err := http.ReadRequest(reader); err != nil { + return + } + + _, _ = fmt.Fprintf(conn, + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s", + len(alpn), alpn) +} + +func selfSignedCert(t *testing.T) tls.Certificate { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "127.0.0.1"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + require.NoError(t, err) + + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +}