[ci] re-run checks

The nightly breakage this worked around is being fixed properly at the
product level (lazy connection warmup, PR #6767), so the harness no
longer needs to force eager peer connections. Until that PR merges,
live e2e runs on this branch will time out in WaitProxyPeer again —
re-run after it lands.

.github/workflows/agent-network-e2e.yml

@@ -51,6 +51,9 @@ jobs:
           # token (and URL, for gateways) is unset, so partial coverage is fine.
           OPENAI_TOKEN: ${{ secrets.E2E_OPENAI_TOKEN }}
           ANTHROPIC_TOKEN: ${{ secrets.E2E_ANTHROPIC_TOKEN }}
+          # Moonshot AI platform key (platform.kimi.ai); drives both Kimi wire
+          # shapes (OpenAI /v1 and Anthropic /anthropic) through kimi_api.
+          KIMI_TOKEN: ${{ secrets.E2E_KIMI_TOKEN }}
           VERCEL_URL: ${{ secrets.E2E_VERCEL_URL }}
           VERCEL_TOKEN: ${{ secrets.E2E_VERCEL_TOKEN }}
           OPENROUTER_URL: ${{ secrets.E2E_OPENROUTER_URL }}

client/internal/peer/worker_relay.go

@@ -54,15 +54,19 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
 	w.relaySupportedOnRemotePeer.Store(true)
 
 	// the relayManager will return with error in case if the connection has lost with relay server
-	_, _, err := w.relayManager.RelayInstanceAddress()
+	currentRelayAddress, _, err := w.relayManager.RelayInstanceAddress()
 	if err != nil {
 		w.log.Errorf("failed to handle new offer: %s", err)
 		return
 	}
 
-	preferForeign := !w.isController
-	remoteRelayServer := relayClient.RelayServer{Addr: remoteOfferAnswer.RelaySrvAddress, IP: remoteOfferAnswer.RelaySrvIP}
-	relayedConn, err := w.relayManager.OpenConn(w.peerCtx, remoteRelayServer, w.config.Key, preferForeign)
+	srv := w.preferredRelayServer(currentRelayAddress, remoteOfferAnswer.RelaySrvAddress)
+	var serverIP netip.Addr
+	if srv == remoteOfferAnswer.RelaySrvAddress {
+		serverIP = remoteOfferAnswer.RelaySrvIP
+	}
+
+	relayedConn, err := w.relayManager.OpenConn(w.peerCtx, srv, w.config.Key, serverIP)
 	if err != nil {
 		if errors.Is(err, relayClient.ErrConnAlreadyExists) {
 			w.log.Debugf("handled offer by reusing existing relay connection")
@@ -76,13 +80,14 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
 	w.relayedConn = relayedConn
 	w.relayLock.Unlock()
 
-	if err := w.relayManager.AddCloseListener(relayedConn.RemoteAddr().String(), w.onRelayClientDisconnected); err != nil {
-		w.log.Errorf("failed to add close listener: %s", err)
+	err = w.relayManager.AddCloseListener(srv, w.onRelayClientDisconnected)
+	if err != nil {
+		log.Errorf("failed to add close listener: %s", err)
 		_ = relayedConn.Close()
 		return
 	}
 
-	w.log.Debugf("peer conn opened via Relay: %s", relayedConn.RemoteAddr())
+	w.log.Debugf("peer conn opened via Relay: %s", srv)
 	go w.conn.onRelayConnectionIsReady(RelayConnInfo{
 		relayedConn:     relayedConn,
 		rosenpassPubKey: remoteOfferAnswer.RosenpassPubKey,
@@ -121,6 +126,13 @@ func (w *WorkerRelay) isRelaySupported(answer *OfferAnswer) bool {
 	return answer.RelaySrvAddress != ""
 }
 
+func (w *WorkerRelay) preferredRelayServer(myRelayAddress, remoteRelayAddress string) string {
+	if w.isController {
+		return myRelayAddress
+	}
+	return remoteRelayAddress
+}
+
 func (w *WorkerRelay) onRelayClientDisconnected() {
 	go w.conn.onRelayDisconnected()
 }

e2e/agentnetwork/chat_test.go

@@ -39,6 +39,21 @@ func availableProviders() []providerCase {
 	if k := os.Getenv("ANTHROPIC_TOKEN"); k != "" {
 		ps = append(ps, providerCase{name: "anthropic", catalogID: "anthropic_api", upstream: "https://api.anthropic.com", apiKey: k, model: "claude-haiku-4-5", kind: harness.WireMessages})
 	}
+	if k := os.Getenv("KIMI_TOKEN"); k != "" {
+		// Kimi (Moonshot AI) serves two body shapes from the same key: OpenAI
+		// Chat Completions on the bare host (/v1/...) and the Anthropic
+		// Messages API under the /anthropic path prefix (the endpoint
+		// Moonshot's Claude Code guide uses). The platform serves this
+		// account exactly ONE model — kimi-k3 (kimi-k2-thinking and even
+		// kimi-latest return resource_not_found_error on both surfaces) — so
+		// two concurrent provider records would claim the same model and
+		// route ambiguously. Run the Anthropic shape, the flagship Claude
+		// Code path; the OpenAI wire shape is covered live by the other
+		// chat-shaped matrix providers, and Kimi-over-chat passed with
+		// kimi-k3 before the single-model constraint surfaced (run #73 on
+		// the kimi feature branch).
+		ps = append(ps, providerCase{name: "kimi", catalogID: "kimi_api", upstream: "https://api.moonshot.ai/anthropic", apiKey: k, model: "kimi-k3", kind: harness.WireMessages})
+	}
 	if k, u := os.Getenv("VERCEL_TOKEN"), os.Getenv("VERCEL_URL"); k != "" && u != "" {
 		ps = append(ps, providerCase{name: "vercel", catalogID: "vercel_ai_gateway", upstream: u, apiKey: k, model: "openai/gpt-4o-mini", kind: harness.WireChat})
 	}
@@ -84,18 +99,38 @@ func availableProviders() []providerCase {
 		}
 	}
 
-	// Bedrock: path-routed, bearer auth. Model is a cross-region inference
-	// profile id (distinct string from the first-party Anthropic case).
+	// Bedrock: path-routed, bearer auth. Model is the FULL cross-region
+	// inference-profile id exactly as AWS issues it — region-family prefix
+	// plus the date/version suffix. A bare or wrong-region id makes Bedrock
+	// reject the request with "The provided model identifier is invalid"
+	// before any inference runs. The proxy normalizes this id to the catalog
+	// key (anthropic.claude-haiku-4-5) for routing/pricing/allowlists.
+	// Defaults pair eu-central-1 with the eu.* profile; AWS_REGION overrides
+	// the region and the prefix follows its family.
 	if k := os.Getenv("AWS_BEARER_TOKEN_BEDROCK"); k != "" {
 		region := os.Getenv("AWS_REGION")
 		if region == "" {
-			region = "us-east-1"
+			region = "eu-central-1"
 		}
-		ps = append(ps, providerCase{name: "bedrock", catalogID: "bedrock_api", upstream: "https://bedrock-runtime." + region + ".amazonaws.com", apiKey: k, model: "us.anthropic.claude-haiku-4-5", kind: harness.WireBedrock})
+		ps = append(ps, providerCase{name: "bedrock", catalogID: "bedrock_api", upstream: "https://bedrock-runtime." + region + ".amazonaws.com", apiKey: k, model: bedrockProfilePrefix(region) + ".anthropic.claude-haiku-4-5-20251001-v1:0", kind: harness.WireBedrock})
 	}
 	return ps
 }
 
+// bedrockProfilePrefix maps an AWS region to its cross-region inference
+// profile prefix: us-east-1 -> us, eu-central-1 -> eu, ap-southeast-1 -> apac,
+// us-gov-west-1 -> us-gov.
+func bedrockProfilePrefix(region string) string {
+	switch {
+	case strings.HasPrefix(region, "us-gov-"):
+		return "us-gov"
+	case strings.HasPrefix(region, "ap-"):
+		return "apac"
+	default:
+		return strings.SplitN(region, "-", 2)[0]
+	}
+}
+
 // providerRequest builds a create request for a matrix provider: enabled, with
 // a uniquely-priced model for body-routed providers and none for the
 // path-routed Vertex (whose model lives in the request path).

e2e/agentnetwork/guardrail_test.go

@@ -4,6 +4,7 @@ package agentnetwork
 
 import (
 	"context"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
@@ -15,13 +16,29 @@ import (
 	"github.com/netbirdio/netbird/shared/management/http/api"
 )
 
+// bedrockVersionSuffix mirrors the proxy's normalizer: the trailing
+// "-vN[:N]" or "-YYYYMMDD-vN[:N]" version/throughput suffix of a Bedrock
+// model id.
+var bedrockVersionSuffix = regexp.MustCompile(`-(\d{8}-)?v\d+(:\d+)?$`)
+
 // catalogModel returns the normalized catalog id the proxy stamps for a
 // path-routed provider's configured model — the form the guardrail allowlist is
 // compared against (region prefix / @version stripped).
 func catalogModel(pc providerCase) string {
 	switch pc.kind {
 	case harness.WireBedrock:
-		return strings.TrimPrefix(pc.model, "us.")
+		// Mirror the proxy's llm.NormalizeBedrockModel (not importable from
+		// here — proxy/internal): strip the region-family prefix and the
+		// date/version suffix so the allowlist carries the catalog key the
+		// guardrail compares against.
+		m := pc.model
+		for _, p := range []string{"us.", "eu.", "apac.", "global."} {
+			if strings.HasPrefix(m, p) {
+				m = strings.TrimPrefix(m, p)
+				break
+			}
+		}
+		return bedrockVersionSuffix.ReplaceAllString(m, "")
 	case harness.WireVertex:
 		return strings.SplitN(pc.model, "@", 2)[0]
 	default:
@@ -35,7 +52,9 @@ func catalogModel(pc providerCase) string {
 func disallowedModel(pc providerCase) string {
 	switch pc.kind {
 	case harness.WireBedrock:
-		return "us.anthropic.claude-opus-4-8"
+		// Same profile prefix as the allowed model so only the model name
+		// differs; the guardrail must deny it before it reaches AWS.
+		return strings.SplitN(pc.model, ".", 2)[0] + ".anthropic.claude-opus-4-8"
 	case harness.WireVertex:
 		return "claude-opus-4-8@20250101"
 	default:

management/internals/modules/agentnetwork/catalog/catalog.go

@@ -420,6 +420,47 @@ var providers = []Provider{
 			{ID: "mistral-embed", Label: "Mistral Embed", InputPer1k: 0.0001, OutputPer1k: 0, ContextWindow: 8192},
 		},
 	},
+	{
+		ID:                 "kimi_api",
+		Kind:               KindProvider,
+		Name:               "Kimi (Moonshot AI) API",
+		Description:        "Kimi K3 / K2 models via the Moonshot AI platform",
+		DefaultHost:        "api.moonshot.ai",
+		AuthHeaderName:     "Authorization",
+		AuthHeaderTemplate: "Bearer ${API_KEY}",
+		DefaultContentType: "application/json",
+		BrandColor:         "#1A1A2E",
+		// ParserID empty on purpose: Moonshot serves two body shapes on
+		// the same host and key, and the proxy's URL sniffer dispatches
+		// both (same pattern as Bifrost). /v1/chat/completions matches
+		// OpenAIParser; the Anthropic-compatible endpoint the official
+		// Claude Code guide uses (/anthropic/v1/messages) contains
+		// "/v1/messages" and matches AnthropicParser. Pinning "openai"
+		// here would misparse the Claude Code path — the primary way
+		// teams consume Kimi for coding today. Both endpoints accept the
+		// same Moonshot key via Authorization: Bearer (Claude Code's
+		// ANTHROPIC_AUTH_TOKEN rides that header too).
+		//
+		// api.moonshot.ai is the international platform; mainland-China
+		// accounts live on api.moonshot.cn with separate billing —
+		// operators there override the host on the provider record. The
+		// kimi.com subscription coding endpoint (api.kimi.com/coding,
+		// model id "k3") is account-bound seat licensing rather than a
+		// meterable platform key, so it's deliberately not the default.
+		ParserID: "",
+		// Pricing per Moonshot's platform rates at K3 launch (July 2026):
+		// $3/$15 per MTok with $0.30 cached input, flat across the 1M-token
+		// window. kimi-k3 is the ONLY model the platform serves newer
+		// accounts — K2-era ids (kimi-k2-thinking) and even the kimi-latest
+		// alias return resource_not_found_error, verified live 2026-07-21 —
+		// so it's the only catalog entry. Grandfathered accounts with K2
+		// access can still type those ids on the provider's model rows.
+		// The consumer app's "K3 Swarm Max" mode is not an API SKU, so it
+		// doesn't appear here.
+		Models: []Model{
+			{ID: "kimi-k3", Label: "Kimi K3", InputPer1k: 0.003, OutputPer1k: 0.015, ContextWindow: 1000000},
+		},
+	},
 	{
 		ID:                 "litellm_proxy",
 		Kind:               KindGateway,

proxy/Dockerfile.multistage

@@ -14,6 +14,7 @@ COPY proxy ./proxy
 COPY route ./route
 COPY shared ./shared
 COPY sharedsock ./sharedsock
+COPY trustedproxy ./trustedproxy
 COPY upload-server ./upload-server
 COPY util ./util
 COPY version ./version

proxy/internal/llm/pricing/defaults_pricing.yaml

@@ -161,6 +161,16 @@ openai:
     input_per_1k: 0.0001
     output_per_1k: 0
 
+  # Kimi / Moonshot AI (kimi_api) — OpenAI-compatible /v1 endpoint. Moonshot
+  # reports cache hits OpenAI-style when present; cached input is 10% of
+  # input ($0.30 vs $3.00 per MTok). kimi-k3 is the only model the platform
+  # serves newer accounts (K2-era ids and kimi-latest 404), matching the
+  # management catalog.
+  kimi-k3:
+    input_per_1k: 0.003
+    output_per_1k: 0.015
+    cached_input_per_1k: 0.0003
+
 anthropic:
   # Claude 4.x family — cache reads ≈10% of input, cache writes ≈125% of input.
   # Pricing source: Anthropic's current published rates per million tokens,
@@ -206,6 +216,20 @@ anthropic:
     cache_read_per_1k: 0.0001
     cache_creation_per_1k: 0.00125
 
+  # Kimi / Moonshot AI (kimi_api) via the Anthropic-compatible endpoint
+  # (/anthropic/v1/messages — the official Claude Code setup). Same rates
+  # as the OpenAI-shape entry above. "kimi-k3[1m]" is the model id some
+  # Claude Code guides set for the 1M-context alias; priced identically so
+  # cost metering doesn't silently skip those requests.
+  kimi-k3:
+    input_per_1k: 0.003
+    output_per_1k: 0.015
+    cache_read_per_1k: 0.0003
+  "kimi-k3[1m]":
+    input_per_1k: 0.003
+    output_per_1k: 0.015
+    cache_read_per_1k: 0.0003
+
 bedrock:
   # AWS Bedrock model ids, normalised by the request parser (cross-region
   # inference-profile prefix + version/throughput suffix stripped), e.g.

shared/relay/client/fallback_opener.go

@@ -1,165 +0,0 @@
-package client
-
-import (
-	"context"
-	"errors"
-	"net"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-)
-
-const (
-	raceTotalTimeout  = 40 * time.Second
-	raceFallbackDelay = 10 * time.Second
-)
-
-type raceAttempt struct {
-	conn net.Conn
-	err  error
-}
-
-type raceOutcome struct {
-	conn net.Conn
-	err  error
-	done bool
-}
-
-type connRace struct {
-	opener            *FallbackOpener
-	peerKey           string
-	remoteRelayServer RelayServer
-	preferForeign     bool
-
-	raceCtx         context.Context
-	otherCtx        context.Context
-	cancelPreferred context.CancelFunc
-	cancelOther     context.CancelFunc
-	results         chan raceAttempt
-	fallbackTimer   *time.Timer
-
-	otherStarted bool
-	settled      int
-	lastErr      error
-}
-
-type FallbackOpener struct {
-	home         *Client
-	foreignStore *ForeignRelaysStore
-}
-
-func NewFallbackOpener(home *Client, foreignStore *ForeignRelaysStore) *FallbackOpener {
-	return &FallbackOpener{
-		home:         home,
-		foreignStore: foreignStore,
-	}
-}
-
-func (r *FallbackOpener) Run(ctx context.Context, peerKey string, remoteRelayServer RelayServer, preferForeign bool) (net.Conn, error) {
-	raceCtx, cancel := context.WithTimeout(ctx, raceTotalTimeout)
-	defer cancel()
-
-	preferredCtx, cancelPreferred := context.WithCancel(raceCtx)
-	otherCtx, cancelOther := context.WithCancel(raceCtx)
-
-	race := &connRace{
-		opener:            r,
-		peerKey:           peerKey,
-		remoteRelayServer: remoteRelayServer,
-		preferForeign:     preferForeign,
-		raceCtx:           raceCtx,
-		otherCtx:          otherCtx,
-		cancelPreferred:   cancelPreferred,
-		cancelOther:       cancelOther,
-		results:           make(chan raceAttempt, 2),
-		fallbackTimer:     time.NewTimer(raceFallbackDelay),
-	}
-	defer race.fallbackTimer.Stop()
-
-	go func() {
-		race.results <- r.open(preferredCtx, peerKey, remoteRelayServer, preferForeign)
-	}()
-
-	for {
-		select {
-		case <-race.fallbackTimer.C:
-			race.startOther()
-		case res := <-race.results:
-			if o := race.handleResult(res); o.done {
-				return o.conn, o.err
-			}
-		case <-raceCtx.Done():
-			return race.onTimeout()
-		}
-	}
-}
-
-func (c *connRace) startOther() {
-	if c.otherStarted {
-		return
-	}
-	c.otherStarted = true
-	c.fallbackTimer.Stop()
-	go func() {
-		c.results <- c.opener.open(c.otherCtx, c.peerKey, c.remoteRelayServer, !c.preferForeign)
-	}()
-}
-
-func (c *connRace) handleResult(res raceAttempt) raceOutcome {
-	if (res.err == nil && res.conn != nil) || errors.Is(res.err, ErrConnAlreadyExists) {
-		c.settled++
-		c.stop()
-		return raceOutcome{conn: res.conn, err: res.err, done: true}
-	}
-
-	c.lastErr = res.err
-	c.settled++
-	if !c.otherStarted {
-		c.startOther()
-		return raceOutcome{}
-	}
-	if c.settled == 2 {
-		c.cancelPreferred()
-		c.cancelOther()
-		return raceOutcome{err: c.lastErr, done: true}
-	}
-	return raceOutcome{}
-}
-
-func (c *connRace) onTimeout() (net.Conn, error) {
-	c.stop()
-	if c.lastErr != nil {
-		return nil, c.lastErr
-	}
-	return nil, c.raceCtx.Err()
-}
-
-func (c *connRace) stop() {
-	c.cancelPreferred()
-	c.cancelOther()
-	go c.opener.drainLoser(c.results, c.settled, c.otherStarted)
-}
-
-func (r *FallbackOpener) open(ctx context.Context, peerKey string, remoteRelayServer RelayServer, foreign bool) raceAttempt {
-	if foreign {
-		conn, err := r.foreignStore.OpenConn(ctx, peerKey, remoteRelayServer)
-		return raceAttempt{conn: conn, err: err}
-	}
-	conn, err := r.home.OpenConn(ctx, peerKey)
-	return raceAttempt{conn: conn, err: err}
-}
-
-func (r *FallbackOpener) drainLoser(results chan raceAttempt, settled int, otherStarted bool) {
-	started := 1
-	if otherStarted {
-		started = 2
-	}
-	for i := settled; i < started; i++ {
-		res := <-results
-		if res.conn != nil {
-			if err := res.conn.Close(); err != nil {
-				log.Debugf("failed to close losing relay connection: %v", err)
-			}
-		}
-	}
-}

shared/relay/client/foreign_relays_store.go

@@ -1,155 +0,0 @@
-package client
-
-import (
-	"context"
-	"net"
-	"sync"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-	"golang.org/x/sync/singleflight"
-
-	relayAuth "github.com/netbirdio/netbird/shared/relay/auth/hmac"
-)
-
-type foreignRelay struct {
-	client  *Client
-	created time.Time
-	inUse   int
-}
-
-type ForeignRelaysStore struct {
-	mu      sync.RWMutex
-	clients map[string]*foreignRelay
-
-	group singleflight.Group
-
-	ctx                  context.Context
-	tokenStore           *relayAuth.TokenStore
-	peerID               string
-	mtu                  uint16
-	transportFallback    *transportFallback
-	onDisconnect         func(string)
-	keepUnusedServerTime time.Duration
-}
-
-func NewForeignRelaysStore(ctx context.Context, tokenStore *relayAuth.TokenStore, peerID string, mtu uint16, transportFallback *transportFallback, onDisconnect func(string), keepUnusedServerTime time.Duration) *ForeignRelaysStore {
-	return &ForeignRelaysStore{
-		clients:              make(map[string]*foreignRelay),
-		ctx:                  ctx,
-		tokenStore:           tokenStore,
-		peerID:               peerID,
-		mtu:                  mtu,
-		transportFallback:    transportFallback,
-		onDisconnect:         onDisconnect,
-		keepUnusedServerTime: keepUnusedServerTime,
-	}
-}
-
-func (f *ForeignRelaysStore) OpenConn(ctx context.Context, peerKey string, remoteRelayServer RelayServer) (net.Conn, error) {
-	fr, err := f.acquire(remoteRelayServer)
-	if err != nil {
-		return nil, err
-	}
-	defer f.release(fr)
-
-	return fr.client.OpenConn(ctx, peerKey)
-}
-
-func (f *ForeignRelaysStore) acquire(remoteRelayServer RelayServer) (*foreignRelay, error) {
-	f.mu.Lock()
-	if fr, ok := f.clients[remoteRelayServer.Addr]; ok {
-		fr.inUse++
-		f.mu.Unlock()
-		return fr, nil
-	}
-	f.mu.Unlock()
-
-	v, err, _ := f.group.Do(remoteRelayServer.Addr, func() (any, error) {
-		f.mu.RLock()
-		fr, ok := f.clients[remoteRelayServer.Addr]
-		f.mu.RUnlock()
-		if ok {
-			return fr, nil
-		}
-
-		relayClient := NewClientWithServerIP(remoteRelayServer.Addr, remoteRelayServer.IP, f.tokenStore, f.peerID, f.mtu)
-		relayClient.SetTransportFallback(f.transportFallback)
-		if err := relayClient.Connect(f.ctx); err != nil {
-			return nil, err
-		}
-		relayClient.SetOnDisconnectListener(f.onDisconnect)
-
-		f.mu.Lock()
-		fr = &foreignRelay{client: relayClient, created: time.Now()}
-		f.clients[remoteRelayServer.Addr] = fr
-		f.mu.Unlock()
-		return fr, nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	fr := v.(*foreignRelay)
-	f.mu.Lock()
-	if cur, ok := f.clients[remoteRelayServer.Addr]; !ok || cur != fr {
-		f.mu.Unlock()
-		return f.acquire(remoteRelayServer)
-	}
-	fr.inUse++
-	f.mu.Unlock()
-	return fr, nil
-}
-
-func (f *ForeignRelaysStore) release(fr *foreignRelay) {
-	f.mu.Lock()
-	fr.inUse--
-	f.mu.Unlock()
-}
-
-func (f *ForeignRelaysStore) evict(serverAddress string) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if _, ok := f.clients[serverAddress]; ok {
-		delete(f.clients, serverAddress)
-		log.Debugf("evicted disconnected foreign relay client: %s", serverAddress)
-	}
-}
-
-func (f *ForeignRelaysStore) cleanupUnused() {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-
-	for addr, fr := range f.clients {
-		if time.Since(fr.created) <= f.keepUnusedServerTime {
-			continue
-		}
-		if fr.inUse > 0 {
-			continue
-		}
-		if fr.client.HasConns() {
-			continue
-		}
-		fr.client.SetOnDisconnectListener(nil)
-		go func() {
-			_ = fr.client.Close()
-		}()
-		log.Debugf("clean up unused relay server connection: %s", addr)
-		delete(f.clients, addr)
-	}
-}
-
-func (f *ForeignRelaysStore) states() []RelayConnState {
-	f.mu.RLock()
-	clients := make([]*Client, 0, len(f.clients))
-	for _, fr := range f.clients {
-		clients = append(clients, fr.client)
-	}
-	f.mu.RUnlock()
-
-	states := make([]RelayConnState, 0, len(clients))
-	for _, c := range clients {
-		states = append(states, relayConnState(c))
-	}
-	return states
-}

shared/relay/client/manager.go

@@ -22,6 +22,27 @@ var (
 	ErrRelayClientNotConnected = fmt.Errorf("relay client not connected")
 )
 
+// RelayTrack hold the relay clients for the foreign relay servers.
+// With the mutex can ensure we can open new connection in case the relay connection has been established with
+// the relay server.
+type RelayTrack struct {
+	sync.RWMutex
+	relayClient *Client
+	err         error
+	created     time.Time
+	// ready is closed once the dial started by openConnVia finishes (relayClient
+	// or err is set). Callers reusing a track wait on this instead of the track
+	// lock, so the dial never runs under rt.Lock.
+	ready chan struct{}
+}
+
+func NewRelayTrack() *RelayTrack {
+	return &RelayTrack{
+		created: time.Now(),
+		ready:   make(chan struct{}),
+	}
+}
+
 type OnServerCloseListener func()
 
 // ManagerOption configures a Manager at construction time.
@@ -38,11 +59,6 @@ type RelayConnState struct {
 	Err error
 }
 
-type RelayServer struct {
-	Addr string
-	IP   netip.Addr
-}
-
 // WithMaxBackoffInterval caps the exponential backoff between reconnect
 // attempts to the home relay. A non-positive value keeps the default.
 func WithMaxBackoffInterval(d time.Duration) ManagerOption {
@@ -67,7 +83,8 @@ type Manager struct {
 	relayClientMu  sync.RWMutex
 	reconnectGuard *Guard
 
-	foreign *ForeignRelaysStore
+	relayClients      map[string]*RelayTrack
+	relayClientsMutex sync.RWMutex
 
 	onDisconnectedListeners map[string]*list.List
 	onReconnectedListenerFn func()
@@ -103,6 +120,7 @@ func NewManager(ctx context.Context, serverURLs []string, peerID string, mtu uin
 			ConnectionTimeout: defaultConnectionTimeout,
 			TransportFallback: tf,
 		},
+		relayClients:            make(map[string]*RelayTrack),
 		onDisconnectedListeners: make(map[string]*list.List),
 		cleanupInterval:         relayCleanupInterval,
 		keepUnusedServerTime:    keepUnusedServerTime,
@@ -110,7 +128,6 @@ func NewManager(ctx context.Context, serverURLs []string, peerID string, mtu uin
 	for _, opt := range opts {
 		opt(m)
 	}
-	m.foreign = NewForeignRelaysStore(ctx, tokenStore, peerID, mtu, tf, m.onServerDisconnected, m.keepUnusedServerTime)
 	m.serverPicker.ServerURLs.Store(serverURLs)
 	m.reconnectGuard = NewGuard(m.serverPicker, m.maxBackoffInterval)
 	return m
@@ -142,26 +159,40 @@ func (m *Manager) Serve() error {
 	return err
 }
 
-func (m *Manager) OpenConn(ctx context.Context, remoteRelayServer RelayServer, peerKey string, preferForeign bool) (net.Conn, error) {
+// OpenConn opens a connection to the given peer key. If the peer is on the same relay server, the connection will be
+// established via the relay server. If the peer is on a different relay server, the manager will establish a new
+// connection to the relay server. It returns back with a net.Conn what represent the remote peer connection.
+//
+// serverIP, when valid and serverAddress is foreign, is used as a dial target if the FQDN-based dial fails.
+// Ignored for the local home-server path. TLS verification still uses the FQDN via SNI.
+func (m *Manager) OpenConn(ctx context.Context, serverAddress, peerKey string, serverIP netip.Addr) (net.Conn, error) {
 	m.relayClientMu.RLock()
-	relayClient := m.relayClient
-	m.relayClientMu.RUnlock()
+	defer m.relayClientMu.RUnlock()
 
-	if relayClient == nil {
+	if m.relayClient == nil {
 		return nil, ErrRelayClientNotConnected
 	}
 
-	foreign, err := m.isForeignServer(relayClient, remoteRelayServer.Addr)
+	foreign, err := m.isForeignServer(serverAddress)
 	if err != nil {
 		return nil, err
 	}
 
+	var (
+		netConn net.Conn
+	)
 	if !foreign {
-		return relayClient.OpenConn(ctx, peerKey)
+		log.Debugf("open peer connection via permanent server: %s", peerKey)
+		netConn, err = m.relayClient.OpenConn(ctx, peerKey)
+	} else {
+		log.Debugf("open peer connection via foreign server: %s", serverAddress)
+		netConn, err = m.openConnVia(ctx, serverAddress, peerKey, serverIP)
+	}
+	if err != nil {
+		return nil, err
 	}
 
-	opener := NewFallbackOpener(relayClient, m.foreign)
-	return opener.Run(ctx, peerKey, remoteRelayServer, preferForeign)
+	return netConn, err
 }
 
 // Ready returns true if the home Relay client is connected to the relay server.
@@ -192,7 +223,7 @@ func (m *Manager) AddCloseListener(serverAddress string, onClosedListener OnServ
 		return ErrRelayClientNotConnected
 	}
 
-	foreign, err := m.isForeignServer(m.relayClient, serverAddress)
+	foreign, err := m.isForeignServer(serverAddress)
 	if err != nil {
 		return err
 	}
@@ -256,7 +287,26 @@ func (m *Manager) RelayStates() []RelayConnState {
 		states = append(states, st)
 	}
 
-	states = append(states, m.foreign.states()...)
+	// Snapshot the tracks, then query each outside the map lock: a track can be
+	// held by an in-progress Connect, and blocking on it must not stall other
+	// relay operations.
+	m.relayClientsMutex.RLock()
+	tracks := make([]*RelayTrack, 0, len(m.relayClients))
+	for _, rt := range m.relayClients {
+		tracks = append(tracks, rt)
+	}
+	m.relayClientsMutex.RUnlock()
+
+	// Only connected foreign relays carry state; a failed connect is evicted
+	// immediately (openConnVia), so there is no error state to surface.
+	for _, rt := range tracks {
+		rt.RLock()
+		rc := rt.relayClient
+		rt.RUnlock()
+		if rc != nil {
+			states = append(states, relayConnState(rc))
+		}
+	}
 
 	return states
 }
@@ -277,6 +327,76 @@ func (m *Manager) UpdateToken(token *relayAuth.Token) error {
 	return m.tokenStore.UpdateToken(token)
 }
 
+func (m *Manager) openConnVia(ctx context.Context, serverAddress, peerKey string, serverIP netip.Addr) (net.Conn, error) {
+	// check if already has a connection to the desired relay server
+	m.relayClientsMutex.RLock()
+	rt, ok := m.relayClients[serverAddress]
+	m.relayClientsMutex.RUnlock()
+	if ok {
+		return m.openConnOnTrack(ctx, rt, peerKey)
+	}
+
+	// if not, establish a new connection but check it again (because changed the lock type) before starting the
+	// connection
+	m.relayClientsMutex.Lock()
+	rt, ok = m.relayClients[serverAddress]
+	if ok {
+		m.relayClientsMutex.Unlock()
+		return m.openConnOnTrack(ctx, rt, peerKey)
+	}
+
+	// Publish the track and release the map lock BEFORE dialing, so the dial does
+	// not run under rt.Lock (which would block RelayStates and the cleanup loop
+	// for the full dial). Concurrent callers find this track and wait on rt.ready.
+	rt = NewRelayTrack()
+	m.relayClients[serverAddress] = rt
+	m.relayClientsMutex.Unlock()
+
+	relayClient := NewClientWithServerIP(serverAddress, serverIP, m.tokenStore, m.peerID, m.mtu)
+	relayClient.SetTransportFallback(m.transportFallback)
+	err := relayClient.Connect(m.ctx)
+	if err != nil {
+		rt.Lock()
+		rt.err = err
+		rt.Unlock()
+		close(rt.ready)
+		m.relayClientsMutex.Lock()
+		delete(m.relayClients, serverAddress)
+		m.relayClientsMutex.Unlock()
+		return nil, err
+	}
+	// if connection closed then delete the relay client from the list
+	relayClient.SetOnDisconnectListener(m.onServerDisconnected)
+	rt.Lock()
+	rt.relayClient = relayClient
+	rt.Unlock()
+	close(rt.ready)
+
+	return relayClient.OpenConn(ctx, peerKey)
+}
+
+// openConnOnTrack opens a peer connection through an existing relay track,
+// waiting for the dial started by another openConnVia call to finish. It waits
+// on rt.ready rather than the track lock, so it neither holds nor contends the
+// track lock across the dial.
+func (m *Manager) openConnOnTrack(ctx context.Context, rt *RelayTrack, peerKey string) (net.Conn, error) {
+	select {
+	case <-rt.ready:
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+
+	rt.RLock()
+	defer rt.RUnlock()
+	if rt.err != nil {
+		return nil, rt.err
+	}
+	if rt.relayClient == nil {
+		return nil, ErrRelayClientNotConnected
+	}
+	return rt.relayClient.OpenConn(ctx, peerKey)
+}
+
 func (m *Manager) onServerConnected() {
 	m.listenerLock.Lock()
 	defer m.listenerLock.Unlock()
@@ -302,12 +422,21 @@ func (m *Manager) onServerDisconnected(serverAddress string) {
 	m.relayClientMu.Unlock()
 
 	if !isHome {
-		m.foreign.evict(serverAddress)
+		m.evictForeignRelay(serverAddress)
 	}
 
 	m.notifyOnDisconnectListeners(serverAddress)
 }
 
+func (m *Manager) evictForeignRelay(serverAddress string) {
+	m.relayClientsMutex.Lock()
+	defer m.relayClientsMutex.Unlock()
+	if _, ok := m.relayClients[serverAddress]; ok {
+		delete(m.relayClients, serverAddress)
+		log.Debugf("evicted disconnected foreign relay client: %s", serverAddress)
+	}
+}
+
 func (m *Manager) listenGuardEvent(ctx context.Context) {
 	for {
 		select {
@@ -330,8 +459,8 @@ func (m *Manager) storeClient(client *Client) {
 	m.relayClient.SetOnDisconnectListener(m.onServerDisconnected)
 }
 
-func (m *Manager) isForeignServer(relayClient *Client, address string) (bool, error) {
-	rAddr, err := relayClient.ServerInstanceURL()
+func (m *Manager) isForeignServer(address string) (bool, error) {
+	rAddr, err := m.relayClient.ServerInstanceURL()
 	if err != nil {
 		return false, fmt.Errorf("relay client not connected")
 	}
@@ -346,11 +475,50 @@ func (m *Manager) startCleanupLoop() {
 		case <-m.ctx.Done():
 			return
 		case <-ticker.C:
-			m.foreign.cleanupUnused()
+			m.cleanUpUnusedRelays()
 		}
 	}
 }
 
+func (m *Manager) cleanUpUnusedRelays() {
+	m.relayClientsMutex.Lock()
+	defer m.relayClientsMutex.Unlock()
+
+	for addr, rt := range m.relayClients {
+		rt.Lock()
+		// if the connection failed to the server the relay client will be nil
+		// but the instance will be kept in the relayClients until the next locking
+		if rt.err != nil {
+			rt.Unlock()
+			continue
+		}
+
+		// dial still in progress (openConnVia publishes the track before Connect
+		// completes and no longer holds rt.Lock during it), nothing to clean up.
+		if rt.relayClient == nil {
+			rt.Unlock()
+			continue
+		}
+
+		if time.Since(rt.created) <= m.keepUnusedServerTime {
+			rt.Unlock()
+			continue
+		}
+
+		if rt.relayClient.HasConns() {
+			rt.Unlock()
+			continue
+		}
+		rt.relayClient.SetOnDisconnectListener(nil)
+		go func() {
+			_ = rt.relayClient.Close()
+		}()
+		log.Debugf("clean up unused relay server connection: %s", addr)
+		delete(m.relayClients, addr)
+		rt.Unlock()
+	}
+}
+
 func (m *Manager) addListener(serverAddress string, onClosedListener OnServerCloseListener) {
 	m.listenerLock.Lock()
 	defer m.listenerLock.Unlock()

shared/relay/client/manager_cleanup_test.go

@@ -2,14 +2,17 @@ package client
 
 import (
 	"context"
+	"net/netip"
 	"testing"
 	"time"
+
+	"github.com/stretchr/testify/require"
 )
 
 // TestCleanUpUnusedRelays_DoesNotBlockOnRealHangingDial drives a real, hanging foreign
-// relay dial and asserts the foreign store cleanup does not stall behind it.
+// relay dial and asserts cleanUpUnusedRelays does not stall behind it.
 func TestCleanUpUnusedRelays_DoesNotBlockOnRealHangingDial(t *testing.T) {
-	serverAddr, accepted := stallingRelayListener(t)
+	serverAddr := stallingRelayListener(t)
 
 	mCtx, mCancel := context.WithCancel(context.Background())
 	t.Cleanup(mCancel)
@@ -19,32 +22,39 @@ func TestCleanUpUnusedRelays_DoesNotBlockOnRealHangingDial(t *testing.T) {
 	dialDone := make(chan struct{})
 	go func() {
 		defer close(dialDone)
-		_, _ = m.foreign.OpenConn(mCtx, "peerKey", RelayServer{Addr: serverAddr})
+		_, _ = m.openConnVia(mCtx, serverAddr, "peerKey", netip.Addr{})
 	}()
 
-	select {
-	case <-accepted:
-	case <-time.After(5 * time.Second):
-		t.Fatal("relay dial did not reach the listener")
-	}
+	// The track appears in the map once the dial is in flight.
+	require.Eventually(t, func() bool {
+		m.relayClientsMutex.RLock()
+		defer m.relayClientsMutex.RUnlock()
+		_, ok := m.relayClients[serverAddr]
+		return ok
+	}, 5*time.Second, 5*time.Millisecond, "relay dial did not start")
 
 	cleanupDone := make(chan struct{})
 	go func() {
 		defer close(cleanupDone)
-		m.foreign.cleanupUnused()
+		m.cleanUpUnusedRelays()
 	}()
 
 	select {
 	case <-cleanupDone:
 	case <-time.After(2 * time.Second):
-		t.Fatal("cleanupUnused blocked on an in-progress relay dial")
+		t.Fatal("cleanUpUnusedRelays blocked on an in-progress relay dial while holding the relay map lock")
 	}
 
+	m.relayClientsMutex.RLock()
+	_, stillTracked := m.relayClients[serverAddr]
+	m.relayClientsMutex.RUnlock()
+	require.True(t, stillTracked, "an in-progress relay dial must not be evicted by cleanup")
+
 	// Release the hanging dial so the goroutine can exit cleanly.
 	mCancel()
 	select {
 	case <-dialDone:
 	case <-time.After(5 * time.Second):
-		t.Fatal("foreign OpenConn did not return after context cancellation")
+		t.Fatal("openConnVia did not return after context cancellation")
 	}
 }

shared/relay/client/manager_relaystates_test.go

@@ -3,6 +3,7 @@ package client
 import (
 	"context"
 	"net"
+	"net/netip"
 	"sync"
 	"testing"
 	"time"
@@ -12,16 +13,13 @@ import (
 
 // stallingRelayListener accepts TCP connections and holds them open without ever
 // responding, so a relay handshake dialed against it blocks until its context is
-// cancelled. accepted is signalled once per incoming connection so a caller can
-// wait until a dial has actually reached the listener. It returns the
-// "rel://host:port" URL to dial.
-func stallingRelayListener(t *testing.T) (string, <-chan struct{}) {
+// cancelled. It returns the "rel://host:port" URL to dial.
+func stallingRelayListener(t *testing.T) string {
 	t.Helper()
 
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	require.NoError(t, err)
 
-	accepted := make(chan struct{}, 1)
 	var mu sync.Mutex
 	var conns []net.Conn
 	go func() {
@@ -33,10 +31,6 @@ func stallingRelayListener(t *testing.T) (string, <-chan struct{}) {
 			mu.Lock()
 			conns = append(conns, c)
 			mu.Unlock()
-			select {
-			case accepted <- struct{}{}:
-			default:
-			}
 		}
 	}()
 	t.Cleanup(func() {
@@ -48,14 +42,14 @@ func stallingRelayListener(t *testing.T) (string, <-chan struct{}) {
 		mu.Unlock()
 	})
 
-	return "rel://" + ln.Addr().String(), accepted
+	return "rel://" + ln.Addr().String()
 }
 
 // TestRelayStates_DoesNotBlockOnRealHangingDial is a regression test for
 // RelayStates() called by a "status -d command" hanging behind an in-progress
-// foreign relay dial.
+// relay dial.
 func TestRelayStates_DoesNotBlockOnRealHangingDial(t *testing.T) {
-	serverAddr, accepted := stallingRelayListener(t)
+	serverAddr := stallingRelayListener(t)
 
 	mCtx, mCancel := context.WithCancel(context.Background())
 	t.Cleanup(mCancel)
@@ -65,14 +59,15 @@ func TestRelayStates_DoesNotBlockOnRealHangingDial(t *testing.T) {
 	dialDone := make(chan struct{})
 	go func() {
 		defer close(dialDone)
-		_, _ = m.foreign.OpenConn(mCtx, "peerKey", RelayServer{Addr: serverAddr})
+		_, _ = m.openConnVia(mCtx, serverAddr, "peerKey", netip.Addr{})
 	}()
 
-	select {
-	case <-accepted:
-	case <-time.After(5 * time.Second):
-		t.Fatal("relay dial did not reach the listener")
-	}
+	require.Eventually(t, func() bool {
+		m.relayClientsMutex.RLock()
+		defer m.relayClientsMutex.RUnlock()
+		_, ok := m.relayClients[serverAddr]
+		return ok
+	}, 5*time.Second, 5*time.Millisecond, "relay dial did not start")
 
 	done := make(chan []RelayConnState, 1)
 	go func() {
@@ -91,6 +86,6 @@ func TestRelayStates_DoesNotBlockOnRealHangingDial(t *testing.T) {
 	select {
 	case <-dialDone:
 	case <-time.After(5 * time.Second):
-		t.Fatal("foreign OpenConn did not return after context cancellation")
+		t.Fatal("openConnVia did not return after context cancellation")
 	}
 }

shared/relay/client/manager_serverip_test.go

@@ -3,6 +3,7 @@ package client
 import (
 	"context"
 	"io"
+	"net/netip"
 	"testing"
 	"time"
 
@@ -84,7 +85,7 @@ func TestManager_ForeignRelayServerIP(t *testing.T) {
 	t.Run("no server IP, dial fails", func(t *testing.T) {
 		dialCtx, dialCancel := context.WithTimeout(ctx, 5*time.Second)
 		defer dialCancel()
-		_, err := mgrAlice.OpenConn(dialCtx, RelayServer{Addr: brokenFQDN}, "bob", true)
+		_, err := mgrAlice.OpenConn(dialCtx, brokenFQDN, "bob", netip.Addr{})
 		if err == nil {
 			t.Fatalf("expected OpenConn to fail without server IP, got success")
 		}
@@ -94,7 +95,7 @@ func TestManager_ForeignRelayServerIP(t *testing.T) {
 		// Bob waits for Alice's incoming peer connection on his side.
 		bobSideCh := make(chan error, 1)
 		go func() {
-			conn, err := mgrBob.OpenConn(ctx, RelayServer{Addr: bobRealAddr}, "alice", false)
+			conn, err := mgrBob.OpenConn(ctx, bobRealAddr, "alice", netip.Addr{})
 			if err != nil {
 				bobSideCh <- err
 				return
@@ -112,7 +113,7 @@ func TestManager_ForeignRelayServerIP(t *testing.T) {
 			bobSideCh <- nil
 		}()
 
-		aliceConn, err := mgrAlice.OpenConn(ctx, RelayServer{Addr: brokenFQDN, IP: bobAdvertisedIP}, "bob", true)
+		aliceConn, err := mgrAlice.OpenConn(ctx, brokenFQDN, "bob", bobAdvertisedIP)
 		if err != nil {
 			t.Fatalf("alice OpenConn with server IP: %s", err)
 		}

shared/relay/client/manager_test.go

@@ -3,6 +3,7 @@ package client
 import (
 	"context"
 	"fmt"
+	"net/netip"
 	"testing"
 	"time"
 
@@ -105,11 +106,11 @@ func TestForeignConn(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to get relay address: %s", err)
 	}
-	connAliceToBob, err := clientAlice.OpenConn(ctx, RelayServer{Addr: bobsSrvAddr}, "bob", true)
+	connAliceToBob, err := clientAlice.OpenConn(ctx, bobsSrvAddr, "bob", netip.Addr{})
 	if err != nil {
 		t.Fatalf("failed to bind channel: %s", err)
 	}
-	connBobToAlice, err := clientBob.OpenConn(ctx, RelayServer{Addr: bobsSrvAddr}, "alice", false)
+	connBobToAlice, err := clientBob.OpenConn(ctx, bobsSrvAddr, "alice", netip.Addr{})
 	if err != nil {
 		t.Fatalf("failed to bind channel: %s", err)
 	}
@@ -209,7 +210,7 @@ func TestForeginConnClose(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to serve manager: %s", err)
 	}
-	conn, err := mgr.OpenConn(ctx, RelayServer{Addr: toURL(srvCfg2)[0]}, "bob", true)
+	conn, err := mgr.OpenConn(ctx, toURL(srvCfg2)[0], "bob", netip.Addr{})
 	if err != nil {
 		t.Fatalf("failed to bind channel: %s", err)
 	}
@@ -301,7 +302,7 @@ func TestForeignAutoClose(t *testing.T) {
 	}
 
 	t.Log("open connection to another peer")
-	if _, err = mgr.OpenConn(ctx, RelayServer{Addr: foreignServerURL}, "anotherpeer", true); err == nil {
+	if _, err = mgr.OpenConn(ctx, foreignServerURL, "anotherpeer", netip.Addr{}); err == nil {
 		t.Fatalf("should have failed to open connection to another peer")
 	}
 
@@ -371,7 +372,7 @@ func TestAutoReconnect(t *testing.T) {
 	if err != nil {
 		t.Errorf("failed to get relay address: %s", err)
 	}
-	conn, err := clientAlice.OpenConn(ctx, RelayServer{Addr: ra}, "bob", false)
+	conn, err := clientAlice.OpenConn(ctx, ra, "bob", netip.Addr{})
 	if err != nil {
 		t.Errorf("failed to bind channel: %s", err)
 	}
@@ -391,7 +392,7 @@ func TestAutoReconnect(t *testing.T) {
 	}
 
 	log.Infof("reopent the connection")
-	_, err = clientAlice.OpenConn(ctx, RelayServer{Addr: ra}, "bob", false)
+	_, err = clientAlice.OpenConn(ctx, ra, "bob", netip.Addr{})
 	if err != nil {
 		t.Errorf("failed to open channel: %s", err)
 	}
@@ -453,7 +454,7 @@ func TestNotifierDoubleAdd(t *testing.T) {
 		t.Fatalf("failed to serve manager: %s", err)
 	}
 
-	conn1, err := clientAlice.OpenConn(ctx, RelayServer{Addr: clientAlice.ServerURLs()[0]}, "bob", false)
+	conn1, err := clientAlice.OpenConn(ctx, clientAlice.ServerURLs()[0], "bob", netip.Addr{})
 	if err != nil {
 		t.Fatalf("failed to bind channel: %s", err)
 	}