netbird/proxy/internal/roundtrip/upstream.go

package roundtrip

import (
	"errors"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
)

// upstreamDowngradeTTL is how long an upstream stays pinned to HTTP/1.1
// after an h2 failure that only implied it cannot serve h2. Bounded so a
// fixed or replaced backend returns to h2 without restarting the proxy.
// A pin the upstream asked for itself does not expire — see downgrade.
const upstreamDowngradeTTL = 10 * time.Minute

// downgrade is an upstream's HTTP/1.1 pin.
type downgrade struct {
	// expiry is when the pin lapses and the upstream is offered h2
	// again. The zero time means it never does: the upstream answered
	// HTTP_1_1_REQUIRED, which is a statement about how it is
	// configured, not a fault that may clear on its own. Re-probing
	// that every upstreamDowngradeTTL would buy nothing but a failed
	// request per interval, so the pin holds until the transport goes
	// away with the proxy or the account's client.
	expiry time.Time
}

// permanent reports whether the upstream asked for this pin itself.
func (d downgrade) permanent() bool {
	return d.expiry.IsZero()
}

// active reports whether the pin still stands at now.
func (d downgrade) active(now time.Time) bool {
	return d.permanent() || now.Before(d.expiry)
}

// upstreamTransport carries requests to a single upstream family (one
// TLS configuration) and implements what upstreamHTTPAuto means.
//
// ALPN already lets the upstream pick the protocol: primary offers both
// h2 and http/1.1 and the server chooses. What ALPN cannot express is
// an upstream that selects h2 and then fails to speak it — the case
// this type handles. The first h2-level failure for a host pins that
// host to fallback, an HTTP/1.1-only clone of primary, and the request
// is retried there when it can be replayed.
//
// The downgrade is per upstream host, not per transport: one broken
// backend must not drop every other backend to HTTP/1.1.
type upstreamTransport struct {
	// primary is the configured transport: h2 offered in ALPN for
	// upstreamHTTPAuto and upstreamHTTP2, HTTP/1.1-only for
	// upstreamHTTP11.
	primary *http.Transport
	// version decides whether a downgrade may happen at all. Only
	// upstreamHTTPAuto downgrades; the explicit values are absolute.
	version upstreamHTTPVersion
	logger  *log.Logger

	// fallbackMu guards the lazy fallback clone: most deployments never
	// hit a broken h2 upstream and should not pay for a second
	// connection pool.
	fallbackMu sync.Mutex
	fallback   *http.Transport

	mu sync.RWMutex
	// downgraded maps an upstream host to its HTTP/1.1 pin.
	downgraded map[string]downgrade
}

// newUpstreamTransport wraps base for the requested HTTP version. base
// must not be used directly afterwards: the wrapper owns it, including
// its connection pool.
func newUpstreamTransport(base *http.Transport, version upstreamHTTPVersion, logger *log.Logger) *upstreamTransport {
	if logger == nil {
		logger = log.StandardLogger()
	}
	applyUpstreamHTTPVersion(base, version)

	return &upstreamTransport{
		primary:    base,
		version:    version,
		logger:     logger,
		downgraded: make(map[string]downgrade),
	}
}

// RoundTrip implements http.RoundTripper.
func (t *upstreamTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	if !t.mayDowngrade(req) {
		return t.primary.RoundTrip(req)
	}

	host := upstreamKey(req.URL)
	if t.isDowngraded(host) {
		return t.http1().RoundTrip(req)
	}

	resp, err := t.primary.RoundTrip(req)
	if err == nil || !isHTTP2ProtocolError(err) {
		return resp, err
	}

	// HTTP_1_1_REQUIRED is the upstream saying it will not serve this
	// request over h2 however often it is asked — IIS answers it for
	// Windows Authentication and for client-certificate sites, where
	// the cause is site configuration rather than a passing fault.
	t.markDowngraded(host, isHTTP11Required(err))

	if !safeToRetry(req, err) {
		// The upstream may have carried out the request before failing
		// to answer over h2, and repeating it could duplicate whatever
		// it did. The host is pinned either way, so the next request
		// goes out over HTTP/1.1.
		t.logger.WithFields(log.Fields{
			"upstream": host,
			"method":   req.Method,
		}).Debug("not retrying over HTTP/1.1: the upstream may already have applied this request")
		return nil, err
	}

	retry, ok := replayable(req)
	if !ok {
		// The body is already consumed and cannot be regenerated, so
		// this request fails, and the pin carries the next one.
		return nil, err
	}
	return t.http1().RoundTrip(retry)
}

// CloseIdleConnections closes idle connections on both pools.
func (t *upstreamTransport) CloseIdleConnections() {
	t.primary.CloseIdleConnections()
	if fallback := t.existingHTTP1(); fallback != nil {
		fallback.CloseIdleConnections()
	}
}

// mayDowngrade reports whether a failed request is a downgrade
// candidate. Only upstreamHTTPAuto downgrades, and only for TLS
// upstreams: the proxy speaks no h2c, so a cleartext upstream is
// already on HTTP/1.1 and an error there says nothing about h2.
func (t *upstreamTransport) mayDowngrade(req *http.Request) bool {
	return t.version == upstreamHTTPAuto && req.URL != nil && req.URL.Scheme == "https"
}

func (t *upstreamTransport) isDowngraded(host string) bool {
	t.mu.RLock()
	pin, ok := t.downgraded[host]
	t.mu.RUnlock()

	if !ok {
		return false
	}
	if pin.active(time.Now()) {
		return true
	}

	t.mu.Lock()
	defer t.mu.Unlock()

	// Re-read under the write lock rather than trusting the expired pin
	// from above: a concurrent request may have re-pinned the host since,
	// and that pin decides this request too. Reporting the stale read
	// would send one request back to h2 against a live pin.
	pin, ok = t.downgraded[host]
	if !ok {
		return false
	}
	if pin.active(time.Now()) {
		return true
	}
	delete(t.downgraded, host)

	return false
}

// markDowngraded pins host to HTTP/1.1. permanent marks a pin the
// upstream asked for; anything else lapses after upstreamDowngradeTTL so
// a repaired backend is offered h2 again.
func (t *upstreamTransport) markDowngraded(host string, permanent bool) {
	now := time.Now()
	pin := downgrade{expiry: now.Add(upstreamDowngradeTTL)}
	if permanent {
		pin = downgrade{}
	}

	t.mu.Lock()
	previous, pinned := t.downgraded[host]
	// A permanent pin is never weakened back into an expiring one: the
	// upstream has already said h2 is not on offer.
	promoted := pinned && !previous.permanent() && permanent
	if !pinned || !previous.permanent() {
		t.downgraded[host] = pin
	}
	for h, existing := range t.downgraded {
		if !existing.active(now) {
			delete(t.downgraded, h)
		}
	}
	t.mu.Unlock()

	// Log a new pin, and a pin the upstream has since asked to make
	// permanent — otherwise an operator would only ever see the "for the
	// next 10m" line and never learn the upstream settled the question.
	if pinned && !promoted {
		return
	}

	entry := t.logger.WithField("upstream", host)
	if permanent {
		entry.Warnf("upstream answered HTTP_1_1_REQUIRED, using HTTP/1.1 for it from now on")
		return
	}
	entry.Warnf("upstream negotiated HTTP/2 but failed to serve it, using HTTP/1.1 for the next %s (set %s=1.1 to pin it)",
		upstreamDowngradeTTL, EnvUpstreamHTTPVersion)
}

// http1 returns the HTTP/1.1-only clone, creating it on first use.
func (t *upstreamTransport) http1() *http.Transport {
	t.fallbackMu.Lock()
	defer t.fallbackMu.Unlock()

	if t.fallback == nil {
		fallback := t.primary.Clone()
		applyUpstreamHTTPVersion(fallback, upstreamHTTP11)
		t.fallback = fallback
	}

	return t.fallback
}

// existingHTTP1 returns the fallback transport only if it was already
// created, so housekeeping never allocates a second connection pool for
// an upstream that never needed one.
func (t *upstreamTransport) existingHTTP1() *http.Transport {
	t.fallbackMu.Lock()
	defer t.fallbackMu.Unlock()

	return t.fallback
}

// upstreamKey normalizes an authority for use as a pin key, so one
// upstream cannot end up with two independent pins. DNS labels compare
// case-insensitively, and the default HTTPS port is implied — every
// downgrade path is TLS-only, so a bare host and the same host on :443
// are the same upstream.
func upstreamKey(u *url.URL) string {
	host := strings.ToLower(u.Hostname())

	port := u.Port()
	if port == "" || port == "443" {
		return host
	}

	// JoinHostPort rather than concatenation: an IPv6 literal needs its
	// brackets back after Hostname stripped them.
	return net.JoinHostPort(host, port)
}

// safeToRetry reports whether req may be sent a second time over
// HTTP/1.1 after err ended its h2 attempt.
//
// A failure at the h2 layer does not say whether the upstream already
// carried out the request, so replaying one that changes state could
// duplicate it. Two cases are safe: a request whose repetition is
// harmless by definition, and an upstream that told us it processed
// nothing on the connection. The second is what makes the IIS case work
// for every method — a site requiring HTTP/1.1 refuses at stream 0,
// before the request is looked at.
func safeToRetry(req *http.Request, err error) bool {
	return idempotent(req) || upstreamProcessedNothing(err)
}

// idempotent reports whether repeating req is defined to be harmless.
// It mirrors net/http's own retry rule (Request.isReplayable): a method
// with no side effects, or a caller that promised the upstream
// deduplicates by key.
func idempotent(req *http.Request) bool {
	if req.Header.Get("Idempotency-Key") != "" || req.Header.Get("X-Idempotency-Key") != "" {
		return true
	}

	switch req.Method {
	// An empty method means GET, as in net/http.
	case "", http.MethodGet, http.MethodHead, http.MethodOptions, http.MethodTrace:
		return true
	}

	return false
}

// upstreamProcessedNothing reports whether err describes a GOAWAY that
// named this request's stream as one the upstream had not received, so
// it cannot have acted on it. A stream error says the opposite: the
// stream was open, so the request had been delivered.
func upstreamProcessedNothing(err error) bool {
	if err == nil {
		return false
	}

	msg := transportError(err).Error()

	return strings.Contains(msg, goAwayStreamNotReceivedMarker) ||
		strings.Contains(msg, goAwayNothingProcessedMarker)
}

// replayable returns a request that can be sent a second time, or
// ok=false when the body is gone. A RoundTripper consumes and closes
// the body it was given, so a retry needs either no body at all or
// GetBody to produce a fresh one.
func replayable(req *http.Request) (*http.Request, bool) {
	if req.Body == nil || req.Body == http.NoBody {
		return req, true
	}
	if req.GetBody == nil {
		return nil, false
	}

	body, err := req.GetBody()
	if err != nil {
		return nil, false
	}

	retry := req.Clone(req.Context())
	retry.Body = body

	return retry, true
}

// http2ErrorMarkers are the substrings that identify an HTTP/2 protocol
// failure. net/http bundles its own private copy of the http2 package,
// so its errors cannot be matched by type from here: http2.StreamError
// and friends in x/net are different types from the ones a
// bundled-h2 transport returns. The strings below are the formats those
// bundled errors print, and they are specific to h2 framing — a
// downgrade must never be triggered by an ordinary network or TLS
// error, which retrying on HTTP/1.1 would not fix.
var http2ErrorMarkers = []string{
	// Transport-level h2 failures, e.g.
	// "http2: server sent GOAWAY and closed the connection".
	"http2:",
	// http2.StreamError, e.g. "stream error: stream ID 1; PROTOCOL_ERROR".
	"stream error: stream ID",
	// http2.ConnectionError, e.g. "connection error: PROTOCOL_ERROR".
	"connection error: ",
	// The code an upstream sends to say the request must be retried
	// over HTTP/1.1, as a GOAWAY or on the stream.
	http11RequiredMarker,
}

const (
	// http11RequiredMarker is the error code an upstream sends to say the
	// request belongs on HTTP/1.1. Unlike the other markers it is not a
	// fault: the upstream is describing its own configuration.
	http11RequiredMarker = "HTTP_1_1_REQUIRED"

	// A GOAWAY carrying NO_ERROR closes a connection without complaint:
	// a server draining before shutdown, recycling an application pool,
	// capping requests per connection. The upstream speaks h2 perfectly
	// well, so this must never pin it. The two spellings are the two
	// formats the bundled transport prints the code in.
	goAwayNoErrorEqualsMarker = "ErrCode=NO_ERROR"
	goAwayNoErrorColonMarker  = "ErrCode:NO_ERROR"
	// gracefulGoAwayMarker is errClientConnGotGoAway, which the bundled
	// transport raises for a stream the server never received on a
	// connection it is shutting down gracefully. It normally retries
	// those itself on a new connection and this never surfaces.
	gracefulGoAwayMarker = "Transport received Server's graceful shutdown GOAWAY"

	// goAwayStreamNotReceivedMarker is the bundled transport's abort for
	// the first stream on a connection whose GOAWAY carried a real error
	// code — the IIS case. It sits in the same "streamID > LastStreamID"
	// branch as the graceful abort, so the server had not received the
	// stream (see net/http's h2_bundle.go).
	goAwayStreamNotReceivedMarker = "Transport received GOAWAY from server ErrCode:"
	// goAwayNothingProcessedMarker is a GoAwayError naming stream 0 as
	// the last one received, which says the same thing. The trailing
	// comma keeps it from matching LastStreamID=10 and the rest.
	goAwayNothingProcessedMarker = "LastStreamID=0,"
)

// isHTTP11Required reports whether the upstream itself asked for
// HTTP/1.1, rather than merely failing at h2.
func isHTTP11Required(err error) bool {
	return err != nil && strings.Contains(transportError(err).Error(), http11RequiredMarker)
}

// isHTTP2ProtocolError reports whether err says the upstream cannot
// serve the h2 it negotiated.
func isHTTP2ProtocolError(err error) bool {
	if err == nil {
		return false
	}

	msg := transportError(err).Error()

	// A graceful GOAWAY is routine connection management, not an
	// upstream that cannot serve h2.
	if isGracefulGoAway(msg) {
		return false
	}

	for _, marker := range http2ErrorMarkers {
		if strings.Contains(msg, marker) {
			return true
		}
	}

	return false
}

// isGracefulGoAway reports whether msg describes a GOAWAY sent to close
// a healthy connection rather than to report an inability to serve h2.
func isGracefulGoAway(msg string) bool {
	return strings.Contains(msg, goAwayNoErrorEqualsMarker) ||
		strings.Contains(msg, goAwayNoErrorColonMarker) ||
		strings.Contains(msg, gracefulGoAwayMarker)
}

// transportError strips a *url.Error wrapper, which prefixes the request
// URL to the message. Markers are matched as substrings, so a URL left
// in place could classify an ordinary dial or TLS failure as an h2 one
// on the strength of the path alone.
func transportError(err error) error {
	var urlErr *url.Error
	if errors.As(err, &urlErr) && urlErr.Err != nil {
		return urlErr.Err
	}

	return err
}