[proxy] add proxy rate limiter (#7568)

2026-09-22 14:49:10 +02:00 · 2026-09-21 15:55:20 +02:00
parent 314d88252d
commit 6c6298f2ab
10 changed files with 701 additions and 16 deletions
@@ -0,0 +1,26 @@
+# PIN and password authentication limits
+
+PIN and password credentials are accepted only in a POST form body. Query-string
+credentials and credentials on other HTTP methods are ignored.
+
+The proxy permits a burst of five credential checks per account and service,
+then replenishes one check every six seconds (ten per minute). PIN and password
+checks share the same budget. Five failed checks from one client IP in a
+rolling five-minute window block that source for fifteen minutes. In-flight checks
+reserve failure slots; blocked requests do not extend the cooldown. Successful
+authentication clears that source's failure history. Infrastructure failures
+consume the service budget without counting as incorrect credentials.
+
+Throttled requests return HTTP 429 with a `Retry-After` delay in seconds. The
+login page displays that delay. Existing authenticated sessions and other
+authentication methods do not consume these credential budgets.
+
+The client IP comes from the existing trusted-proxy resolution. Deployments
+behind a load balancer must configure trusted proxies correctly; otherwise
+visitors share the load balancer's source budget. Visitors behind the same NAT
+also share a source budget for a service.
+
+State is held in memory per proxy process and resets on restart. Multiple
+replicas have independent budgets. State is bounded to 16,384 source entries and
+4,096 service entries; when capacity is exhausted, new checks are denied until
+idle entries expire. Active blocks are never evicted to admit a new source.
@@ -0,0 +1,100 @@
+package auth
+
+import (
+	"errors"
+	"math"
+	"net/http"
+	"strconv"
+	"time"
+
+	"google.golang.org/genproto/googleapis/rpc/errdetails"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"github.com/netbirdio/netbird/proxy/auth"
+	"github.com/netbirdio/netbird/proxy/internal/proxy"
+)
+
+var errCredentialClientIP = errors.New("invalid client address")
+
+type credentialLimitError struct {
+	retryAfter time.Duration
+}
+
+func (e *credentialLimitError) Error() string {
+	return "too many authentication attempts"
+}
+
+func credentialFormValue(r *http.Request, field string) string {
+	if r.Method != http.MethodPost {
+		return ""
+	}
+	return r.PostFormValue(field)
+}
+
+func (mw *Middleware) authenticateScheme(r *http.Request, config DomainConfig, scheme Scheme) (string, string, error) {
+	method := scheme.Type()
+	if (method != auth.MethodPIN && method != auth.MethodPassword) || !wasCredentialSubmitted(r, method) {
+		return scheme.Authenticate(r)
+	}
+	ip := mw.resolveClientIP(r).Unmap()
+	if !ip.IsValid() {
+		return "", "", errCredentialClientIP
+	}
+	source, retry := mw.credentials.begin(credentialSourceKey{
+		service: credentialServiceKey{accountID: config.AccountID, serviceID: config.ServiceID},
+		ip:      ip,
+	})
+	if retry > 0 {
+		return "", "", &credentialLimitError{retryAfter: retry}
+	}
+	token, prompt, err := scheme.Authenticate(r)
+	outcome := credentialUnavailable
+	if err == nil {
+		outcome = credentialRejected
+		if token != "" {
+			outcome = credentialAccepted
+		}
+	}
+	mw.credentials.finish(source, outcome)
+	return token, prompt, err
+}
+
+func credentialRetryAfter(err error) time.Duration {
+	var limitErr *credentialLimitError
+	if errors.As(err, &limitErr) {
+		return limitErr.retryAfter
+	}
+	s := status.Convert(err)
+	if s.Code() != codes.ResourceExhausted {
+		return 0
+	}
+	for _, detail := range s.Details() {
+		if info, ok := detail.(*errdetails.RetryInfo); ok && info.RetryDelay != nil && info.RetryDelay.CheckValid() == nil {
+			if delay := info.RetryDelay.AsDuration(); delay > 0 {
+				return delay
+			}
+		}
+	}
+	return credentialCheckInterval
+}
+
+func (mw *Middleware) writeAuthenticationError(w http.ResponseWriter, r *http.Request, method auth.Method, err error) {
+	if cd := proxy.CapturedDataFromContext(r.Context()); cd != nil {
+		cd.SetOrigin(proxy.OriginAuth)
+		cd.SetAuthMethod(method.String())
+	}
+	if retry := credentialRetryAfter(err); retry > 0 {
+		// RFC 6585 section 4 forbids caching 429 responses.
+		w.Header().Set("Cache-Control", "no-store")
+		w.Header().Set("Retry-After", strconv.FormatInt(int64(math.Ceil(retry.Seconds())), 10))
+		http.Error(w, "too many authentication attempts; try again later", http.StatusTooManyRequests)
+		return
+	}
+	if errors.Is(err, errCredentialClientIP) {
+		http.Error(w, "invalid client address", http.StatusBadRequest)
+		return
+	}
+	mw.logger.WithField("scheme", method.String()).Warnf("authentication infrastructure error: %v", err)
+	http.Error(w, "authentication service unavailable", http.StatusBadGateway)
+}
@@ -0,0 +1,169 @@
+package auth
+
+import (
+	"net/netip"
+	"sync"
+	"time"
+
+	"golang.org/x/time/rate"
+
+	"github.com/netbirdio/netbird/proxy/internal/types"
+)
+
+const (
+	credentialFailureLimit    = 5
+	credentialFailureWindow   = 5 * time.Minute
+	credentialBlockDuration   = 15 * time.Minute
+	credentialCheckInterval   = 6 * time.Second
+	credentialCheckBurst      = 5
+	credentialMaxSources      = 16384
+	credentialMaxServices     = 4096
+	credentialCleanupInterval = time.Minute
+)
+
+type credentialServiceKey struct {
+	accountID types.AccountID
+	serviceID types.ServiceID
+}
+
+type credentialSourceKey struct {
+	service credentialServiceKey
+	ip      netip.Addr
+}
+
+type credentialSource struct {
+	failures     []time.Time
+	pending      int
+	expiresAt    time.Time
+	blockedUntil time.Time
+}
+
+type credentialService struct {
+	limiter  *rate.Limiter
+	lastUsed time.Time
+}
+
+type credentialOutcome string
+
+const (
+	credentialUnavailable credentialOutcome = "unavailable"
+	credentialRejected    credentialOutcome = "rejected"
+	credentialAccepted    credentialOutcome = "accepted"
+)
+
+// State is local to this proxy process. Active blocks are never evicted to
+// make room for a new source; exhausting capacity denies new checks.
+type credentialLimiter struct {
+	mu          sync.Mutex
+	now         func() time.Time
+	sources     map[credentialSourceKey]*credentialSource
+	services    map[credentialServiceKey]*credentialService
+	nextCleanup time.Time
+}
+
+func newCredentialLimiter() *credentialLimiter {
+	return &credentialLimiter{
+		now:      time.Now,
+		sources:  make(map[credentialSourceKey]*credentialSource),
+		services: make(map[credentialServiceKey]*credentialService),
+	}
+}
+
+func (l *credentialLimiter) begin(key credentialSourceKey) (*credentialSource, time.Duration) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	now := l.now()
+	l.cleanup(now)
+	source := l.sources[key]
+	if source != nil {
+		if now.Before(source.blockedUntil) {
+			return nil, source.blockedUntil.Sub(now)
+		}
+		if source.pending == 0 && !now.Before(source.expiresAt) {
+			*source = credentialSource{}
+		}
+		source.expireFailures(now)
+		// Reserve the failure budget before verification so concurrent guesses
+		// cannot all pass a check against the same completed failure count.
+		if len(source.failures)+source.pending >= credentialFailureLimit {
+			return nil, time.Second
+		}
+	} else if len(l.sources) >= credentialMaxSources {
+		return nil, credentialCleanupInterval
+	}
+	if retry := l.allowService(key.service, now); retry > 0 {
+		return nil, retry
+	}
+	if source == nil {
+		source = &credentialSource{}
+		l.sources[key] = source
+	}
+	if source.expiresAt.IsZero() {
+		source.expiresAt = now.Add(credentialFailureWindow)
+	}
+	source.pending++
+	return source, 0
+}
+
+func (l *credentialLimiter) allowService(key credentialServiceKey, now time.Time) time.Duration {
+	service := l.services[key]
+	if service == nil {
+		if len(l.services) >= credentialMaxServices {
+			return credentialCleanupInterval
+		}
+		service = &credentialService{limiter: rate.NewLimiter(rate.Every(credentialCheckInterval), credentialCheckBurst)}
+		l.services[key] = service
+	}
+	service.lastUsed = now
+	if service.limiter.AllowN(now, 1) {
+		return 0
+	}
+	return max(time.Nanosecond, time.Duration((1-service.limiter.TokensAt(now))*float64(credentialCheckInterval)))
+}
+
+func (l *credentialLimiter) finish(source *credentialSource, outcome credentialOutcome) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	source.pending--
+	now := l.now()
+	source.expireFailures(now)
+	switch outcome {
+	case credentialRejected:
+		source.failures = append(source.failures, now)
+		source.expiresAt = now.Add(credentialFailureWindow)
+		if len(source.failures) >= credentialFailureLimit && source.blockedUntil.IsZero() {
+			source.blockedUntil = now.Add(credentialBlockDuration)
+			source.expiresAt = source.blockedUntil
+		}
+	case credentialAccepted:
+		if !now.Before(source.blockedUntil) {
+			source.failures = nil
+			source.expiresAt = now.Add(credentialFailureWindow)
+		}
+	case credentialUnavailable:
+		// Transport failures consume the service budget, but are not bad guesses.
+	}
+}
+
+func (s *credentialSource) expireFailures(now time.Time) {
+	for len(s.failures) > 0 && !now.Before(s.failures[0].Add(credentialFailureWindow)) {
+		s.failures = s.failures[1:]
+	}
+}
+
+func (l *credentialLimiter) cleanup(now time.Time) {
+	if now.Before(l.nextCleanup) {
+		return
+	}
+	l.nextCleanup = now.Add(credentialCleanupInterval)
+	for key, source := range l.sources {
+		if source.pending == 0 && !now.Before(source.expiresAt) {
+			delete(l.sources, key)
+		}
+	}
+	for key, service := range l.services {
+		if now.Sub(service.lastUsed) >= credentialBlockDuration {
+			delete(l.services, key)
+		}
+	}
+}
@@ -0,0 +1,190 @@
+package auth
+
+import (
+	"net/netip"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/proxy/internal/types"
+)
+
+func TestCredentialLimiterCooldown(t *testing.T) {
+	l := newCredentialLimiter()
+	now := time.Now()
+	l.now = func() time.Time { return now }
+	key := credentialSourceKey{service: credentialServiceKey{"account", "service"}, ip: netip.MustParseAddr("192.0.2.1")}
+	for range credentialFailureLimit {
+		attempt, retry := l.begin(key)
+		require.Zero(t, retry, "initial guesses must reach verification")
+		l.finish(attempt, credentialRejected)
+	}
+	_, retry := l.begin(key)
+	assert.Equal(t, credentialBlockDuration, retry, "five failures must start a fifteen-minute block")
+	now = now.Add(credentialBlockDuration - time.Second)
+	_, retry = l.begin(key)
+	assert.Equal(t, time.Second, retry, "blocked requests must not extend the deadline")
+	now = now.Add(time.Second)
+	attempt, retry := l.begin(key)
+	require.Zero(t, retry, "the source must recover when its block expires")
+	l.finish(attempt, credentialAccepted)
+}
+
+func TestCredentialLimiterFailureWindowAndSuccess(t *testing.T) {
+	for _, outcome := range []credentialOutcome{credentialAccepted, credentialUnavailable} {
+		t.Run(map[credentialOutcome]string{credentialAccepted: "success", credentialUnavailable: "infrastructure error"}[outcome], func(t *testing.T) {
+			l := newCredentialLimiter()
+			now := time.Now()
+			l.now = func() time.Time { return now }
+			key := credentialSourceKey{service: credentialServiceKey{"account", "service"}, ip: netip.MustParseAddr("192.0.2.1")}
+			for range 4 {
+				attempt, retry := l.begin(key)
+				require.Zero(t, retry, "four failures must fit the budget")
+				l.finish(attempt, credentialRejected)
+			}
+			attempt, retry := l.begin(key)
+			require.Zero(t, retry, "fifth check must be allowed")
+			l.finish(attempt, outcome)
+			now = now.Add(credentialCheckInterval)
+			attempt, retry = l.begin(key)
+			require.Zero(t, retry, "success or infrastructure error must not start a block")
+			l.finish(attempt, credentialRejected)
+			now = now.Add(credentialCheckInterval)
+			attempt, retry = l.begin(key)
+			if outcome == credentialUnavailable {
+				assert.Greater(t, retry, time.Duration(0), "infrastructure errors must preserve earlier failures")
+				return
+			}
+			require.Zero(t, retry, "success must clear earlier failures")
+			l.finish(attempt, credentialRejected)
+			now = now.Add(credentialFailureWindow)
+			for range credentialFailureLimit {
+				attempt, retry = l.begin(key)
+				require.Zero(t, retry, "old failures must expire")
+				l.finish(attempt, credentialRejected)
+			}
+		})
+	}
+}
+
+func TestCredentialLimiterRollingWindow(t *testing.T) {
+	l := newCredentialLimiter()
+	now := time.Now()
+	l.now = func() time.Time { return now }
+	key := credentialSourceKey{service: credentialServiceKey{"account", "service"}, ip: netip.MustParseAddr("192.0.2.1")}
+	attempt, retry := l.begin(key)
+	require.Zero(t, retry, "the first failure starts the history")
+	l.finish(attempt, credentialRejected)
+	now = now.Add(4 * time.Minute)
+	for range 3 {
+		attempt, retry = l.begin(key)
+		require.Zero(t, retry, "three more failures must fit the budget")
+		l.finish(attempt, credentialRejected)
+	}
+	now = now.Add(time.Minute + time.Second)
+	for range 2 {
+		attempt, retry = l.begin(key)
+		require.Zero(t, retry, "only the oldest failure must have expired")
+		l.finish(attempt, credentialRejected)
+	}
+	_, retry = l.begin(key)
+	assert.Equal(t, credentialBlockDuration, retry, "five recent failures must block even across the first window boundary")
+}
+
+func TestCredentialLimiterServiceBudget(t *testing.T) {
+	l := newCredentialLimiter()
+	now := time.Now()
+	l.now = func() time.Time { return now }
+	key := credentialSourceKey{service: credentialServiceKey{"account", "service"}, ip: netip.MustParseAddr("192.0.2.1")}
+	for range credentialCheckBurst {
+		attempt, retry := l.begin(key)
+		require.Zero(t, retry, "initial checks must fit the service burst")
+		l.finish(attempt, credentialAccepted)
+		key.ip = key.ip.Next()
+	}
+	_, retry := l.begin(key)
+	assert.Equal(t, credentialCheckInterval, retry, "changing IP must not bypass the service budget")
+	other := key
+	other.service.accountID = "another-account"
+	attempt, retry := l.begin(other)
+	require.Zero(t, retry, "accounts must have separate budgets")
+	l.finish(attempt, credentialAccepted)
+	other = key
+	other.service.serviceID = "another-service"
+	attempt, retry = l.begin(other)
+	require.Zero(t, retry, "services must have separate budgets")
+	l.finish(attempt, credentialAccepted)
+	now = now.Add(credentialCheckInterval)
+	attempt, retry = l.begin(key)
+	require.Zero(t, retry, "one check must refill every six seconds")
+	l.finish(attempt, credentialAccepted)
+	_, retry = l.begin(key)
+	assert.Equal(t, credentialCheckInterval, retry, "refill must only grant one new check")
+}
+
+func TestCredentialLimiterConcurrentReservations(t *testing.T) {
+	l := newCredentialLimiter()
+	now := time.Now()
+	l.now = func() time.Time { return now }
+	key := credentialSourceKey{service: credentialServiceKey{"account", "service"}, ip: netip.MustParseAddr("192.0.2.1")}
+	var attempts []*credentialSource
+	for range credentialFailureLimit {
+		attempt, retry := l.begin(key)
+		require.Zero(t, retry, "initial requests must reserve the failure budget")
+		attempts = append(attempts, attempt)
+	}
+	// Refill the service budget while earlier verification calls are still running.
+	now = now.Add(time.Minute)
+	var admitted atomic.Int32
+	var wg sync.WaitGroup
+	for range 100 {
+		wg.Go(func() {
+			attempt, retry := l.begin(key)
+			if retry == 0 {
+				admitted.Add(1)
+				l.finish(attempt, credentialRejected)
+			}
+		})
+	}
+	wg.Wait()
+	assert.Zero(t, admitted.Load(), "in-flight guesses must reserve the failure budget despite a refilled service budget")
+	for _, attempt := range attempts {
+		wg.Go(func() { l.finish(attempt, credentialRejected) })
+	}
+	wg.Wait()
+	_, retry := l.begin(key)
+	assert.Equal(t, credentialBlockDuration, retry, "concurrent failures must activate the block")
+}
+
+func TestCredentialLimiterCapacityAndCleanup(t *testing.T) {
+	for _, fullSources := range []bool{true, false} {
+		t.Run(map[bool]string{true: "sources", false: "services"}[fullSources], func(t *testing.T) {
+			l := newCredentialLimiter()
+			now := time.Now()
+			l.now = func() time.Time { return now }
+			key := credentialSourceKey{service: credentialServiceKey{"account", "service"}, ip: netip.MustParseAddr("192.0.2.1")}
+			if fullSources {
+				ip := netip.MustParseAddr("198.18.0.1")
+				for range credentialMaxSources {
+					l.sources[credentialSourceKey{service: key.service, ip: ip}] = &credentialSource{expiresAt: now.Add(credentialBlockDuration), blockedUntil: now.Add(credentialBlockDuration)}
+					ip = ip.Next()
+				}
+			} else {
+				for i := range credentialMaxServices {
+					l.services[credentialServiceKey{serviceID: key.service.serviceID, accountID: types.AccountID(strconv.Itoa(i))}] = &credentialService{lastUsed: now}
+				}
+			}
+			_, retry := l.begin(key)
+			assert.Positive(t, retry, "full state must deny new checks without evicting active entries")
+			now = now.Add(credentialBlockDuration)
+			attempt, retry := l.begin(key)
+			require.Zero(t, retry, "expired state must release capacity")
+			l.finish(attempt, credentialAccepted)
+		})
+	}
+}
@@ -0,0 +1,196 @@
+package auth
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/genproto/googleapis/rpc/errdetails"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/durationpb"
+
+	"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service"
+	servicemanager "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service/manager"
+	"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/sessionkey"
+	nbgrpc "github.com/netbirdio/netbird/management/internals/shared/grpc"
+	"github.com/netbirdio/netbird/management/server/store"
+	mgmttypes "github.com/netbirdio/netbird/management/server/types"
+	proxyauth "github.com/netbirdio/netbird/proxy/auth"
+	"github.com/netbirdio/netbird/proxy/internal/proxy"
+	"github.com/netbirdio/netbird/shared/management/proto"
+)
+
+// localCredentialClient replaces the transport while keeping the real service
+// store, credential verification, and session signing.
+type localCredentialClient struct {
+	server *nbgrpc.ProxyServiceServer
+}
+
+func (c localCredentialClient) Authenticate(ctx context.Context, req *proto.AuthenticateRequest, _ ...grpc.CallOption) (*proto.AuthenticateResponse, error) {
+	return c.server.Authenticate(ctx, req)
+}
+
+func credentialHandler(t *testing.T, field string) (*Middleware, http.Handler) {
+	t.Helper()
+	ctx := context.Background()
+	s, err := store.NewStore(ctx, mgmttypes.SqliteStoreEngine, t.TempDir(), nil, false)
+	require.NoError(t, err)
+	t.Cleanup(func() { assert.NoError(t, s.Close(ctx)) })
+	require.NoError(t, s.SaveAccount(ctx, &mgmttypes.Account{Id: "account"}))
+	keys := generateTestKeyPair(t)
+	svc := &service.Service{
+		ID: "service", AccountID: "account", Name: "test", Domain: "example.com",
+		Enabled: true, SessionPrivateKey: keys.PrivateKey, SessionPublicKey: keys.PublicKey,
+		Auth: service.AuthConfig{
+			PinAuth:      &service.PINAuthConfig{Enabled: true, Pin: "842716"},
+			PasswordAuth: &service.PasswordAuthConfig{Enabled: true, Password: "842716"},
+		},
+	}
+	require.NoError(t, svc.Auth.HashSecrets())
+	require.NoError(t, s.CreateService(ctx, svc))
+	server := nbgrpc.NewProxyServiceServer(nil, nil, nil, nbgrpc.ProxyOIDCConfig{}, nil, nil, nil, nil, nil)
+	t.Cleanup(server.Close)
+	server.SetServiceManager(servicemanager.NewManager(s, nil, nil, nil, nil, nil))
+	client := localCredentialClient{server: server}
+	var scheme Scheme = NewPin(client, "service", "account")
+	if field == "password" {
+		scheme = NewPassword(client, "service", "account")
+	}
+	mw := NewMiddleware(nil, nil, nil)
+	require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, keys.PublicKey, time.Hour, "account", "service", nil, false, nil))
+	return mw, mw.Protect(newPassthroughHandler())
+}
+
+func credentialRequest(method, field, value string) *http.Request {
+	r := httptest.NewRequest(method, "https://example.com/", strings.NewReader(url.Values{field: {value}}.Encode()))
+	r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	r.RemoteAddr = "198.51.100.25:12345"
+	return r
+}
+
+func TestCredentialAuthPOSTOnly(t *testing.T) {
+	for _, field := range []string{"pin", "password"} {
+		t.Run(field, func(t *testing.T) {
+			_, handler := credentialHandler(t, field)
+			for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodPatch, http.MethodDelete, http.MethodPost} {
+				r := credentialRequest(method, field, "")
+				r.URL.RawQuery = url.Values{field: {"842716"}}.Encode()
+				resp := httptest.NewRecorder()
+				handler.ServeHTTP(resp, r)
+				assert.Equal(t, http.StatusUnauthorized, resp.Code, "%s query credentials must not authenticate", method)
+				assert.Empty(t, resp.Result().Cookies(), "query credentials must not issue a session")
+			}
+			for _, method := range []string{http.MethodGet, http.MethodPut, http.MethodPatch, http.MethodDelete} {
+				resp := httptest.NewRecorder()
+				handler.ServeHTTP(resp, credentialRequest(method, field, "842716"))
+				assert.Equal(t, http.StatusUnauthorized, resp.Code, "%s body credentials must not authenticate", method)
+			}
+			resp := httptest.NewRecorder()
+			handler.ServeHTTP(resp, credentialRequest(http.MethodPost, field, "842716"))
+			assert.Equal(t, http.StatusSeeOther, resp.Code, "POST body credentials must authenticate")
+		})
+	}
+}
+
+func TestCredentialAuthThrottling(t *testing.T) {
+	for _, field := range []string{"pin", "password"} {
+		t.Run(field, func(t *testing.T) {
+			_, handler := credentialHandler(t, field)
+			for range 5 {
+				resp := httptest.NewRecorder()
+				handler.ServeHTTP(resp, credentialRequest(http.MethodPost, field, "000000"))
+				require.Equal(t, http.StatusUnauthorized, resp.Code, "initial wrong credentials must be rejected")
+			}
+			resp := httptest.NewRecorder()
+			handler.ServeHTTP(resp, credentialRequest(http.MethodPost, field, "842716"))
+			assert.Equal(t, http.StatusTooManyRequests, resp.Code, "even correct credentials must wait for the block to expire")
+			assert.Equal(t, "900", resp.Header().Get("Retry-After"), "five failures must block the source for fifteen minutes")
+			assert.Empty(t, resp.Result().Cookies(), "blocked credentials must not issue a session")
+		})
+	}
+}
+
+func TestCredentialAuthSessionAndClientIP(t *testing.T) {
+	keys := generateTestKeyPair(t)
+	token, err := sessionkey.SignToken(keys.PrivateKey, "pin-user", "", "example.com", proxyauth.MethodPIN, nil, nil, time.Hour)
+	require.NoError(t, err)
+	mw := NewMiddleware(nil, nil, nil)
+	now := time.Now()
+	mw.credentials.now = func() time.Time { return now }
+	scheme := &stubScheme{method: proxyauth.MethodPIN, promptID: "pin"}
+	require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, keys.PublicKey, 0, "account", "service", nil, false, nil))
+	handler := mw.Protect(newPassthroughHandler())
+	for range credentialFailureLimit {
+		resp := httptest.NewRecorder()
+		handler.ServeHTTP(resp, credentialRequest(http.MethodPost, "pin", "000000"))
+		require.Equal(t, http.StatusUnauthorized, resp.Code, "bad PIN must consume the failure budget")
+	}
+	now = now.Add(credentialCheckInterval)
+	require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, keys.PublicKey, 0, "account", "service", nil, false, nil))
+	r := credentialRequest(http.MethodPost, "pin", "000000")
+	r.RemoteAddr = "[::ffff:198.51.100.25]:45678"
+	r.Header.Set("X-Forwarded-For", "192.0.2.5")
+	r.Header.Set("X-Real-IP", "192.0.2.6")
+	resp := httptest.NewRecorder()
+	handler.ServeHTTP(resp, r)
+	assert.Equal(t, http.StatusTooManyRequests, resp.Code, "mapped addresses and untrusted forwarding headers must not bypass the source block")
+	assert.Equal(t, "no-store", resp.Header().Get("Cache-Control"), "rate limits must not be cached")
+	r.AddCookie(&http.Cookie{Name: proxyauth.SessionCookieName, Value: token})
+	resp = httptest.NewRecorder()
+	handler.ServeHTTP(resp, r)
+	assert.Equal(t, http.StatusOK, resp.Code, "an existing session must pass even with credentials in the request")
+	assert.Equal(t, "backend", resp.Body.String(), "the authenticated request must reach the application")
+	r = credentialRequest(http.MethodPost, "pin", "000000")
+	cd := proxy.NewCapturedData("test")
+	cd.SetClientIP(netip.MustParseAddr("192.0.2.9"))
+	r = r.WithContext(proxy.WithCapturedData(r.Context(), cd))
+	resp = httptest.NewRecorder()
+	handler.ServeHTTP(resp, r)
+	assert.Equal(t, http.StatusUnauthorized, resp.Code, "a client resolved by the trusted-proxy middleware must get its own source budget")
+	r = credentialRequest(http.MethodPost, "pin", "000000")
+	r.RemoteAddr = "invalid"
+	resp = httptest.NewRecorder()
+	handler.ServeHTTP(resp, r)
+	assert.Equal(t, http.StatusBadRequest, resp.Code, "an unresolvable client address must fail closed")
+	now = now.Add(credentialBlockDuration)
+	scheme.token = token
+	resp = httptest.NewRecorder()
+	handler.ServeHTTP(resp, credentialRequest(http.MethodPost, "pin", "842716"))
+	assert.Equal(t, http.StatusSeeOther, resp.Code, "credentials must work again after cooldown")
+}
+
+func TestCredentialAuthManagementThrottling(t *testing.T) {
+	s, err := status.New(codes.ResourceExhausted, "rate limited").WithDetails(&errdetails.RetryInfo{RetryDelay: durationpb.New(2500 * time.Millisecond)})
+	require.NoError(t, err)
+	for _, tc := range []struct {
+		name  string
+		err   error
+		code  int
+		retry string
+	}{
+		{"retry info", fmt.Errorf("authenticate PIN: %w", s.Err()), http.StatusTooManyRequests, "3"},
+		{"missing retry info", status.Error(codes.ResourceExhausted, "rate limited"), http.StatusTooManyRequests, "6"},
+		{"unavailable", status.Error(codes.Unavailable, "unavailable"), http.StatusBadGateway, ""},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			keys := generateTestKeyPair(t)
+			mw := NewMiddleware(nil, nil, nil)
+			scheme := &stubScheme{method: proxyauth.MethodPIN, authFn: func(*http.Request) (string, string, error) { return "", "", tc.err }}
+			require.NoError(t, mw.AddDomain("example.com", []Scheme{scheme}, keys.PublicKey, 0, "account", "service", nil, false, nil))
+			resp := httptest.NewRecorder()
+			mw.Protect(newPassthroughHandler()).ServeHTTP(resp, credentialRequest(http.MethodPost, "pin", "000000"))
+			assert.Equal(t, tc.code, resp.Code, "management errors must keep their HTTP meaning")
+			assert.Equal(t, tc.retry, resp.Header().Get("Retry-After"), "retry hints must round up to whole seconds")
+		})
+	}
+}
@@ -87,6 +87,7 @@ type Middleware struct {
 	sessionValidator SessionValidator
 	geo              restrict.GeoResolver
 	tunnelCache      *tunnelValidationCache
+	credentials      *credentialLimiter
 }

 // NewMiddleware creates a new authentication middleware. The sessionValidator is
@@ -101,6 +102,7 @@ func NewMiddleware(logger *log.Logger, sessionValidator SessionValidator, geo re
 		sessionValidator: sessionValidator,
 		geo:              geo,
 		tunnelCache:      newTunnelValidationCache(),
+		credentials:      newCredentialLimiter(),
 	}
 }

@@ -543,13 +545,9 @@ func (mw *Middleware) authenticateWithSchemes(w http.ResponseWriter, r *http.Req
 	var attemptedMethod string

 	for _, scheme := range config.Schemes {
-		token, promptData, err := scheme.Authenticate(r)
+		token, promptData, err := mw.authenticateScheme(r, config, scheme)
 		if err != nil {
-			mw.logger.WithField("scheme", scheme.Type().String()).Warnf("authentication infrastructure error: %v", err)
-			if cd := proxy.CapturedDataFromContext(r.Context()); cd != nil {
-				cd.SetOrigin(proxy.OriginAuth)
-			}
-			http.Error(w, "authentication service unavailable", http.StatusBadGateway)
+			mw.writeAuthenticationError(w, r, scheme.Type(), err)
 			return
 		}

@@ -650,9 +648,9 @@ func setSessionCookie(w http.ResponseWriter, token string, expiration time.Durat
 func wasCredentialSubmitted(r *http.Request, method auth.Method) bool {
 	switch method {
 	case auth.MethodPIN:
-		return r.FormValue("pin") != ""
+		return credentialFormValue(r, pinFormId) != ""
 	case auth.MethodPassword:
-		return r.FormValue("password") != ""
+		return credentialFormValue(r, passwordFormId) != ""
 	case auth.MethodOIDC:
 		return r.URL.Query().Get("session_token") != ""
 	}
@@ -35,7 +35,7 @@ func (Password) Type() auth.Method {
 // so that it can be injected into a request from the UI so that
 // authentication may be successful.
 func (p Password) Authenticate(r *http.Request) (string, string, error) {
-	password := r.FormValue(passwordFormId)
+	password := credentialFormValue(r, passwordFormId)

 	if password == "" {
 		// No password submitted; return the form ID so the UI can prompt the user.
@@ -35,7 +35,7 @@ func (Pin) Type() auth.Method {
 // so that it can be injected into a request from the UI so that
 // authentication may be successful.
 func (p Pin) Authenticate(r *http.Request) (string, string, error) {
-	pin := r.FormValue(pinFormId)
+	pin := credentialFormValue(r, pinFormId)

 	if pin == "" {
 		// No PIN submitted; return the form ID so the UI can prompt the user.
@@ -68,6 +68,12 @@ function App() {
        if (res.type === "opaqueredirect" || res.status === 0) {
          setSubmitting("redirect");
          globalThis.location.reload();
+        } else if (res.status === 429) {
+          const seconds = Number(res.headers.get("Retry-After"));
+          const wait = Number.isFinite(seconds) && seconds > 0
+            ? ` Try again in ${Math.ceil(seconds)} seconds.`
+            : " Please try again later.";
+          handleAuthError(method, `Too many authentication attempts.${wait}`);
        } else {
          handleAuthError(method, "Authentication failed. Please try again.");
        }