[e2e] Add live chat-through-proxy scenario (Pillar 3)

Stand up the full agent-network data path in containers and drive a real
chat-completion through the gateway:

- harness: a shared docker network (combined server reachable by alias),
  a proxy container built from the published reverse-proxy image
  (NB_PROXY_PRIVATE, NB_PROXY_ALLOW_INSECURE, NB_RELAY_TRANSPORT=ws to match
  the combined server's WS-multiplexed relay) with a generated self-signed
  wildcard cert, and a netbird client container that joins via a setup key.
- the combined image, proxy image, and client image default to the
  published rc.2 releases (overridable via NB_E2E_*_IMAGE; a bare local tag
  is built from source instead). Geolocation download is disabled so the
  server starts without external fetches.
- one shared domain is used for the management exposed address, the proxy
  domain, and the agent-network cluster; the proxy token is minted via the
  server CLI (global) to match the manual install.

TestChatCompletionThroughProxy provisions provider+policy+group+setup key,
runs proxy+client, drives an OpenAI chat-completion through the tunnel, and
asserts a 200 plus the ingested access-log row. Requires OPENAI_TOKEN
(skips otherwise). The provider must be created with enabled=true explicitly
— the create default is false despite the API doc.

e2e/agentnetwork/chat_test.go

@@ -0,0 +1,165 @@
+//go:build e2e
+
+package agentnetwork
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/e2e/harness"
+	"github.com/netbirdio/netbird/shared/management/http/api"
+)
+
+// TestChatCompletionThroughProxy is Pillar 3: it provisions an agent-network
+// gateway (provider + policy + setup key), runs the proxy and a client
+// container on the shared network, and drives a real chat-completion from the
+// client through the proxy to the upstream provider over the WireGuard tunnel,
+// asserting a 200 and that usage is recorded.
+//
+// Requires a real provider key in OPENAI_TOKEN (source ~/.llm-keys locally; set
+// the Actions secret in CI). Skips otherwise.
+func TestChatCompletionThroughProxy(t *testing.T) {
+	apiKey := os.Getenv("OPENAI_TOKEN")
+	if apiKey == "" {
+		t.Skip("OPENAI_TOKEN not set; source ~/.llm-keys to run the live chat test")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
+	defer cancel()
+
+	// Group + setup key: the client joins into this group; the policy authorizes
+	// it to reach the provider.
+	grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-agents"})
+	require.NoError(t, err, "create agents group")
+	t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
+
+	ephemeral := false
+	sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
+		Name:       "e2e-client",
+		Type:       "reusable",
+		ExpiresIn:  86400,
+		UsageLimit: 0,
+		AutoGroups: []string{grp.Id},
+		Ephemeral:  &ephemeral,
+	})
+	require.NoError(t, err, "mint setup key")
+	require.NotEmpty(t, sk.Key, "setup key must be returned in plaintext")
+
+	// Provider (real upstream key) + policy authorizing the group. Created
+	// before the proxy starts so the proxy's initial cluster snapshot already
+	// carries the account's synthesized service.
+	prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
+		Name:             "OpenAI Live",
+		ProviderId:       "openai_api",
+		UpstreamUrl:      "https://api.openai.com",
+		ApiKey:           &apiKey,
+		Enabled:          ptr(true),
+		BootstrapCluster: ptr(harness.AgentNetworkCluster),
+		Models: &[]api.AgentNetworkProviderModel{
+			{Id: "gpt-4o-mini", InputPer1k: 0.00015, OutputPer1k: 0.0006},
+		},
+	})
+	require.NoError(t, err, "create provider")
+	t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
+
+	enabled := true
+	policyReq := api.AgentNetworkPolicyRequest{
+		Name:                   "e2e-allow",
+		Enabled:                &enabled,
+		SourceGroups:           []string{grp.Id},
+		DestinationProviderIds: []string{prov.Id},
+	}
+	pol, err := srv.CreatePolicy(ctx, policyReq)
+	require.NoError(t, err, "create policy")
+	t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
+
+	settings, err := srv.GetSettings(ctx)
+	require.NoError(t, err, "read settings for endpoint")
+	require.NotEmpty(t, settings.Endpoint, "agent-network endpoint must be assigned")
+
+	// Mint the proxy token via the server CLI (global, account-less) — the path
+	// the manual install uses, which drives the cluster-snapshot synthesis the
+	// proxy needs. An account-scoped REST token takes a different path that
+	// doesn't deliver the service.
+	proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-proxy")
+	require.NoError(t, err, "mint proxy token via CLI")
+
+	px, err := harness.StartProxy(ctx, srv, proxyToken)
+	require.NoError(t, err, "start proxy")
+	t.Cleanup(func() { _ = px.Terminate(context.Background()) })
+
+	// Client joins last, once the proxy + provider + policy are all in place, so
+	// its initial network map includes the synthesized agent-network service.
+	cl, err := harness.StartClient(ctx, srv, sk.Key)
+	require.NoError(t, err, "start client")
+	t.Cleanup(func() { _ = cl.Terminate(context.Background()) })
+
+	require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management")
+
+	if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil {
+		dctx := context.Background()
+		peers, _ := srv.API().Peers.List(dctx)
+		var peerInfo []string
+		for _, p := range peers {
+			var groups []string
+			for _, g := range p.Groups {
+				groups = append(groups, g.Name)
+			}
+			peerInfo = append(peerInfo, fmt.Sprintf("%s connected=%t ip=%s groups=%v", p.Name, p.Connected, p.Ip, groups))
+		}
+		clusters, _ := srv.API().ReverseProxyClusters.List(dctx)
+		var clusterInfo []string
+		for _, cl := range clusters {
+			clusterInfo = append(clusterInfo, fmt.Sprintf("%+v", cl))
+		}
+		domains, _ := srv.API().ReverseProxyDomains.List(dctx)
+		var domainInfo []string
+		for _, d := range domains {
+			domainInfo = append(domainInfo, fmt.Sprintf("%+v", d))
+		}
+		_ = os.WriteFile("/tmp/nb-e2e-proxy.log", []byte(px.Logs(dctx)), 0o644)
+		_ = os.WriteFile("/tmp/nb-e2e-client.log", []byte(cl.Logs(dctx)), 0o644)
+		_ = os.WriteFile("/tmp/nb-e2e-combined.log", []byte(srv.Logs(dctx)), 0o644)
+		diag := fmt.Sprintf("settings: cluster=%q endpoint=%q subdomain=%q\nprovider: id=%s cluster=%s\npolicy: id=%s sourceGroups=%v dst=%v\ngroup: id=%s\npeers:\n%s\nclusters:\n%s\n",
+			settings.Cluster, settings.Endpoint, settings.Subdomain,
+			prov.Id, harness.AgentNetworkCluster,
+			pol.Id, policyReq.SourceGroups, policyReq.DestinationProviderIds,
+			grp.Id,
+			strings.Join(peerInfo, "\n"), strings.Join(clusterInfo, "\n"))
+		diag += "domains:\n" + strings.Join(domainInfo, "\n") + "\n"
+		_ = os.WriteFile("/tmp/nb-e2e-diag.txt", []byte(diag), 0o644)
+		t.Fatalf("client did not see the proxy peer: %v\n=== settings ===\ncluster=%q endpoint=%q subdomain=%q\n=== peers ===\n%v\n=== clusters ===\n%v\n=== proxy logs ===\n%s",
+			err, settings.Cluster, settings.Endpoint, settings.Subdomain, peerInfo, clusterInfo, px.Logs(dctx))
+	}
+
+	proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint)
+	require.NoError(t, err, "resolve agent-network endpoint to proxy IP")
+
+	code, body, err := cl.Chat(ctx, settings.Endpoint, proxyIP, "gpt-4o-mini", "Reply with exactly: pong")
+	require.NoError(t, err, "chat request through tunnel")
+	if code != 200 {
+		t.Fatalf("expected 200 from chat-completion, got %d\nbody: %s\n=== proxy logs ===\n%s", code, body, px.Logs(context.Background()))
+	}
+	assert.Contains(t, body, "choices", "chat response should carry choices")
+
+	// The per-request access-log row is ingested asynchronously after the
+	// response is forwarded; poll briefly. (Consumption rows are only booked
+	// when a policy has token/budget limits, which this one doesn't.)
+	require.Eventually(t, func() bool {
+		resp, lerr := srv.ListAccessLogs(ctx)
+		return lerr == nil && resp.TotalRecords > 0
+	}, 30*time.Second, 2*time.Second, "an access-log row should be recorded after the chat-completion")
+
+	logs, err := srv.ListAccessLogs(ctx)
+	require.NoError(t, err, "read access logs")
+	require.NotEmpty(t, logs.Data, "access-log page must contain the request row")
+	require.NotNil(t, logs.Data[0].Model, "access-log row should record the model")
+	assert.Equal(t, "gpt-4o-mini", *logs.Data[0].Model, "access-log row should record the requested model")
+}

e2e/harness/Dockerfile.client

@@ -0,0 +1,24 @@
+# Multistage build for the NetBird client used in e2e tests. The repo has no
+# source-building client Dockerfile (client/Dockerfile packages a goreleaser
+# artifact), so this mirrors its alpine runtime + entrypoint while compiling the
+# CGO-free client inline. BuildKit cache mounts keep rebuilds incremental.
+
+FROM golang:1.25-bookworm AS builder
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN --mount=type=cache,target=/go/pkg/mod go mod download
+COPY . .
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    CGO_ENABLED=0 GOOS=linux go build -o /out/netbird ./client
+
+FROM alpine:3.24
+RUN apk add --no-cache bash ca-certificates ip6tables iproute2 iptables
+ENV NETBIRD_BIN="/usr/local/bin/netbird" \
+    NB_LOG_FILE="console,/var/log/netbird/client.log" \
+    NB_DAEMON_ADDR="unix:///var/run/netbird.sock" \
+    NB_ENABLE_CAPTURE="false" \
+    NB_ENTRYPOINT_SERVICE_TIMEOUT="30"
+ENTRYPOINT [ "/usr/local/bin/netbird-entrypoint.sh" ]
+COPY client/netbird-entrypoint.sh /usr/local/bin/netbird-entrypoint.sh
+COPY --from=builder /out/netbird /usr/local/bin/netbird

e2e/harness/agentnetwork.go

@@ -79,6 +79,11 @@ func (c *Combined) CreatePolicy(ctx context.Context, req api.AgentNetworkPolicyR
 	return anRequest[api.AgentNetworkPolicy](ctx, c, http.MethodPost, "/api/agent-network/policies", req)
 }
 
+// UpdatePolicy replaces a policy by id.
+func (c *Combined) UpdatePolicy(ctx context.Context, id string, req api.AgentNetworkPolicyRequest) (api.AgentNetworkPolicy, error) {
+	return anRequest[api.AgentNetworkPolicy](ctx, c, http.MethodPut, "/api/agent-network/policies/"+id, req)
+}
+
 // DeletePolicy removes a policy by id.
 func (c *Combined) DeletePolicy(ctx context.Context, id string) error {
 	return anDelete(ctx, c, "/api/agent-network/policies/"+id)
@@ -99,3 +104,9 @@ func (c *Combined) UpdateSettings(ctx context.Context, req api.AgentNetworkSetti
 func (c *Combined) ListConsumption(ctx context.Context) ([]api.AgentNetworkConsumption, error) {
 	return anRequest[[]api.AgentNetworkConsumption](ctx, c, http.MethodGet, "/api/agent-network/consumption", nil)
 }
+
+// ListAccessLogs returns the account's agent-network access-log page (the
+// flattened per-request rows the proxy ships and management ingests).
+func (c *Combined) ListAccessLogs(ctx context.Context) (api.AgentNetworkAccessLogsResponse, error) {
+	return anRequest[api.AgentNetworkAccessLogsResponse](ctx, c, http.MethodGet, "/api/agent-network/access-logs", nil)
+}

e2e/harness/bootstrap.go

@@ -21,7 +21,7 @@ func (c *Combined) Bootstrap(ctx context.Context) (string, error) {
 
 	createPAT := true
 	expireDays := 1
-	resp, err := setupClient.Instance.Setup(ctx, api.PostApiSetupJSONRequestBody{
+	resp, err := setupClient.Instance.Setup(ctx, api.PostApiSetupJSONRequestBody{ //nolint:gosec // static throwaway test credentials
 		Email:       "admin@netbird.test",
 		Password:    "Netbird-e2e-Passw0rd!",
 		Name:        "E2E Admin",

e2e/harness/cert.go

@@ -0,0 +1,64 @@
+//go:build e2e
+
+package harness
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"math/big"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+// writeSelfSignedCert generates a self-signed TLS cert/key pair covering the
+// given DNS names and writes them as tls.crt / tls.key in dir. The proxy serves
+// this for the agent-network endpoint; the client curls with -k, so validity
+// chains don't matter — the proxy just needs a usable cert to present.
+func writeSelfSignedCert(dir string, dnsNames []string) error {
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return fmt.Errorf("generate key: %w", err)
+	}
+
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return fmt.Errorf("generate serial: %w", err)
+	}
+
+	tmpl := x509.Certificate{
+		SerialNumber:          serial,
+		Subject:               pkix.Name{CommonName: dnsNames[0]},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(365 * 24 * time.Hour),
+		KeyUsage:              x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		DNSNames:              dnsNames,
+		BasicConstraintsValid: true,
+	}
+
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
+	if err != nil {
+		return fmt.Errorf("create certificate: %w", err)
+	}
+
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	if err := os.WriteFile(filepath.Join(dir, "tls.crt"), certPEM, 0o644); err != nil { //nolint:gosec // public cert, bind-mounted and read by the proxy container
+		return fmt.Errorf("write cert: %w", err)
+	}
+
+	keyDER, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return fmt.Errorf("marshal key: %w", err)
+	}
+	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
+	if err := os.WriteFile(filepath.Join(dir, "tls.key"), keyPEM, 0o600); err != nil {
+		return fmt.Errorf("write key: %w", err)
+	}
+	return nil
+}

e2e/harness/client.go

@@ -0,0 +1,207 @@
+//go:build e2e
+
+package harness
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/testcontainers/testcontainers-go"
+	tcexec "github.com/testcontainers/testcontainers-go/exec"
+)
+
+const (
+	clientDockerfile = "e2e/harness/Dockerfile.client"
+	// defaultClientImage is the published NetBird client release used by
+	// default. Override with NB_E2E_CLIENT_IMAGE; a value without a "/" is built
+	// locally from clientDockerfile.
+	defaultClientImage = "netbirdio/netbird:0.74.0-rc.2"
+	clientAlias        = "client"
+	curlImage          = "curlimages/curl:latest"
+)
+
+// Client is a running NetBird client container joined to the combined server.
+type Client struct {
+	container testcontainers.Container
+}
+
+// StartClient builds the client image and runs it on the combined server's
+// network, joining via the given setup key. The image entrypoint brings the
+// daemon up automatically; callers wait for connectivity with WaitConnected /
+// WaitProxyPeer.
+func StartClient(ctx context.Context, c *Combined, setupKey string) (*Client, error) {
+	root, err := repoRoot()
+	if err != nil {
+		return nil, err
+	}
+	clientImage, err := resolveImage(ctx, root, "NB_E2E_CLIENT_IMAGE", defaultClientImage, clientDockerfile)
+	if err != nil {
+		return nil, err
+	}
+
+	req := testcontainers.ContainerRequest{
+		Image:          clientImage,
+		Networks:       []string{c.network.Name},
+		NetworkAliases: map[string][]string{c.network.Name: {clientAlias}},
+		Env: map[string]string{
+			"NB_MANAGEMENT_URL": combinedExposedURL,
+			"NB_SETUP_KEY":      setupKey,
+			"NB_LOG_LEVEL":      "info",
+			// Match the proxy: the combined relay is WebSocket-only, so the
+			// client must use WS transport to keep a stable relay link to it.
+			"NB_RELAY_TRANSPORT": "ws",
+		},
+		HostConfigModifier: func(hc *container.HostConfig) {
+			hc.CapAdd = append(hc.CapAdd, "NET_ADMIN", "SYS_ADMIN", "SYS_RESOURCE")
+		},
+	}
+
+	ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: req,
+		Started:          true,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("start client container: %w", err)
+	}
+	return &Client{container: ctr}, nil
+}
+
+// Restart bounces the client connection (netbird down/up) so it pulls a fresh
+// network map — the documented workaround for a freshly-joined client not yet
+// seeing a synthesized agent-network service.
+func (cl *Client) Restart(ctx context.Context) error {
+	if _, _, err := cl.container.Exec(ctx, []string{"netbird", "down"}, tcexec.Multiplexed()); err != nil {
+		return fmt.Errorf("netbird down: %w", err)
+	}
+	time.Sleep(2 * time.Second)
+	code, reader, err := cl.container.Exec(ctx, []string{"netbird", "up"}, tcexec.Multiplexed())
+	if err != nil {
+		return fmt.Errorf("netbird up: %w", err)
+	}
+	if code != 0 {
+		out, _ := io.ReadAll(reader)
+		return fmt.Errorf("netbird up exited %d: %s", code, string(out))
+	}
+	return nil
+}
+
+// Status returns `netbird status` output from inside the client.
+func (cl *Client) Status(ctx context.Context) (string, error) {
+	code, reader, err := cl.container.Exec(ctx, []string{"netbird", "status"}, tcexec.Multiplexed())
+	if err != nil {
+		return "", err
+	}
+	out, _ := io.ReadAll(reader)
+	if code != 0 {
+		return string(out), fmt.Errorf("netbird status exited %d", code)
+	}
+	return string(out), nil
+}
+
+// WaitConnected polls until the client reports Management: Connected.
+func (cl *Client) WaitConnected(ctx context.Context, timeout time.Duration) error {
+	return cl.pollStatus(ctx, timeout, "Management: Connected")
+}
+
+// WaitProxyPeer polls until the client sees the proxy peer connected (1/1).
+func (cl *Client) WaitProxyPeer(ctx context.Context, timeout time.Duration) error {
+	return cl.pollStatus(ctx, timeout, "1/1 Connected")
+}
+
+func (cl *Client) pollStatus(ctx context.Context, timeout time.Duration, want string) error {
+	deadline := time.Now().Add(timeout)
+	var last string
+	for time.Now().Before(deadline) {
+		out, _ := cl.Status(ctx)
+		last = out
+		if strings.Contains(out, want) {
+			return nil
+		}
+		time.Sleep(3 * time.Second)
+	}
+	return fmt.Errorf("timed out waiting for %q; last status:\n%s", want, last)
+}
+
+// ResolveProxyIP resolves the agent-network endpoint to the proxy peer's
+// NetBird IP from inside the client (via magic DNS).
+func (cl *Client) ResolveProxyIP(ctx context.Context, endpoint string) (string, error) {
+	code, reader, err := cl.container.Exec(ctx, []string{"getent", "hosts", endpoint}, tcexec.Multiplexed())
+	if err != nil {
+		return "", err
+	}
+	out, _ := io.ReadAll(reader)
+	if code != 0 {
+		return "", fmt.Errorf("getent hosts %s exited %d", endpoint, code)
+	}
+	fields := strings.Fields(string(out))
+	if len(fields) == 0 {
+		return "", fmt.Errorf("no address for %s", endpoint)
+	}
+	return fields[0], nil
+}
+
+// Chat issues a chat-completion POST to the agent-network endpoint over the
+// client's tunnel, returning the HTTP status and response body. It runs curl in
+// a throwaway container sharing the client's network namespace so the request
+// traverses the WireGuard tunnel, pinning the endpoint to the proxy peer IP.
+func (cl *Client) Chat(ctx context.Context, endpoint, proxyIP, model, prompt string) (int, string, error) {
+	body := fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":%q}]}`, model, prompt)
+	url := "https://" + endpoint + "/v1/chat/completions"
+
+	args := []string{
+		"run", "--rm",
+		"--network", "container:" + cl.container.GetContainerID(),
+		curlImage,
+		"-sk", "--connect-timeout", "5", "--max-time", "90",
+		"--resolve", endpoint + ":443:" + proxyIP,
+		"-o", "/dev/stderr", "-w", "%{http_code}",
+		"-X", "POST", url,
+		"-H", "Content-Type: application/json",
+		"--data", body,
+	}
+	cmd := exec.CommandContext(ctx, "docker", args...)
+	// -w writes the status code to stdout; -o /dev/stderr writes the body to
+	// stderr so we can capture both separately.
+	var stdout, stderr strings.Builder
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		return 0, stderr.String(), fmt.Errorf("curl through tunnel: %w", err)
+	}
+
+	code := 0
+	_, _ = fmt.Sscanf(strings.TrimSpace(stdout.String()), "%d", &code)
+	return code, stderr.String(), nil
+}
+
+// Logs returns the client container logs, for diagnostics on failure.
+func (cl *Client) Logs(ctx context.Context) string {
+	return containerLogs(ctx, cl.container)
+}
+
+// Terminate stops the client container.
+func (cl *Client) Terminate(ctx context.Context) error {
+	if cl.container == nil {
+		return nil
+	}
+	return cl.container.Terminate(ctx)
+}
+
+// containerLogs reads up to 256 KiB of a container's logs for diagnostics.
+func containerLogs(ctx context.Context, c testcontainers.Container) string {
+	if c == nil {
+		return ""
+	}
+	r, err := c.Logs(ctx)
+	if err != nil {
+		return fmt.Sprintf("<logs error: %v>", err)
+	}
+	defer r.Close()
+	b, _ := io.ReadAll(io.LimitReader(r, 256<<10))
+	return string(b)
+}

e2e/harness/combined.go

@@ -5,14 +5,18 @@ package harness
 import (
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/go-connections/nat"
 	"github.com/testcontainers/testcontainers-go"
+	tcexec "github.com/testcontainers/testcontainers-go/exec"
+	"github.com/testcontainers/testcontainers-go/network"
 	"github.com/testcontainers/testcontainers-go/wait"
 
 	"github.com/netbirdio/netbird/shared/management/client/rest"
@@ -20,20 +24,34 @@ import (
 
 const (
 	combinedDockerfile = "combined/Dockerfile.multistage"
-	combinedImageTag   = "netbird-e2e-combined:latest"
+	// defaultCombinedImage is the published combined-server release used by
-	combinedHTTPPort   = "8080/tcp"
+	// default (the artifact operators run manually). Override with
+	// NB_E2E_COMBINED_IMAGE; a value without a "/" is built locally from
+	// combinedDockerfile instead of pulled.
+	defaultCombinedImage = "netbirdio/netbird-server:0.74.0-rc.2"
+	combinedHTTPPort     = "8080/tcp"
 
-	// containerInternalURL is how the server addresses itself: it listens on
+	// combinedAlias is the combined server's network alias AND the deployment
-	// :8080 inside the container, so the embedded IdP issuer and exposed
+	// domain. The working manual setup uses a single NETBIRD_DOMAIN for the
-	// address both resolve there. Tests reach it via the mapped host port.
+	// management exposed address, the proxy domain, and the agent-network
-	containerInternalURL = "http://localhost:8080"
+	// cluster — so we mirror that: peers reach management/signal/relay at this
-	containerIssuer      = containerInternalURL + "/oauth2"
+	// name, the proxy registers this as its cluster, and the agent-network
+	// endpoint is <subdomain>.<combinedAlias>.
+	combinedAlias      = "netbird.local"
+	combinedExposedURL = "http://" + combinedAlias + ":8080"
+
+	// containerIssuer is the embedded IdP issuer, used only for internal JWT
+	// validation (peers authenticate with setup keys / proxy tokens, not OIDC),
+	// so the in-container localhost address is fine.
+	containerIssuer = "http://localhost:8080/oauth2"
 )
 
 // Combined is a running combined NetBird server (management + signal + relay +
-// STUN + embedded IdP) plus the connection details tests need.
+// STUN + embedded IdP) plus the connection details tests need. It owns the
+// shared docker network that the proxy and client containers join.
 type Combined struct {
 	container testcontainers.Container
+	network   *testcontainers.DockerNetwork
 	// BaseURL is the host-reachable management API root, e.g. http://127.0.0.1:51234.
 	BaseURL string
 	// PAT is the admin Personal Access Token minted via Bootstrap.
@@ -44,41 +62,52 @@ type Combined struct {
 }
 
 // StartCombined builds the combined server from its multistage Dockerfile and
-// boots it with setup-PAT enabled, returning once the API is serving. The
+// boots it with setup-PAT enabled on a fresh shared network, returning once the
-// caller still owns minting the admin PAT via Bootstrap.
+// API is serving. The caller still owns minting the admin PAT via Bootstrap.
 func StartCombined(ctx context.Context) (*Combined, error) {
 	root, err := repoRoot()
 	if err != nil {
 		return nil, err
 	}
 
-	// Build with the docker CLI (BuildKit) rather than testcontainers' classic
+	combinedImage, err := resolveImage(ctx, root, "NB_E2E_COMBINED_IMAGE", defaultCombinedImage, combinedDockerfile)
-	// build path so the Dockerfile's cache mounts are honored and recompiles
+	if err != nil {
-	// stay incremental. testcontainers then just runs the tagged image.
-	if err := buildCombinedImage(ctx, root); err != nil {
 		return nil, err
 	}
 
+	net, err := network.New(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("create shared network: %w", err)
+	}
+
 	// Work dir under /tmp so Docker Desktop file sharing (which excludes
 	// macOS's /var/folders TMPDIR) can bind-mount it.
 	workDir, err := os.MkdirTemp("/tmp", "nb-e2e-combined-*")
 	if err != nil {
+		_ = net.Remove(ctx)
 		return nil, fmt.Errorf("create work dir: %w", err)
 	}
 
-	cfg := fmt.Sprintf(combinedConfigYAML, containerInternalURL, containerIssuer)
+	cfg := fmt.Sprintf(combinedConfigYAML, combinedExposedURL, containerIssuer)
-	if err := os.WriteFile(filepath.Join(workDir, "config.yaml"), []byte(cfg), 0o644); err != nil {
+	if err := os.WriteFile(filepath.Join(workDir, "config.yaml"), []byte(cfg), 0o644); err != nil { //nolint:gosec // non-secret config, bind-mounted and read by the container
+		_ = net.Remove(ctx)
 		return nil, fmt.Errorf("write combined config: %w", err)
 	}
 	if err := os.MkdirAll(filepath.Join(workDir, "data"), 0o755); err != nil {
+		_ = net.Remove(ctx)
 		return nil, fmt.Errorf("create datadir: %w", err)
 	}
 
 	req := testcontainers.ContainerRequest{
-		Image:        combinedImageTag,
+		Image:          combinedImage,
-		ExposedPorts: []string{combinedHTTPPort},
+		ExposedPorts:   []string{combinedHTTPPort},
+		Networks:       []string{net.Name},
+		NetworkAliases: map[string][]string{net.Name: {combinedAlias}},
 		Env: map[string]string{
 			"NB_SETUP_PAT_ENABLED": "true",
+			// Skip the GeoLite DB download — it blocks startup and agent-network
+			// ingest doesn't use geolocation.
+			"NB_DISABLE_GEOLOCATION": "true",
 		},
 		Cmd: []string{"--config", "/nb/config.yaml"},
 		HostConfigModifier: func(hc *container.HostConfig) {
@@ -87,7 +116,7 @@ func StartCombined(ctx context.Context) (*Combined, error) {
 		WaitingFor: wait.ForHTTP("/api/instance").
 			WithPort(combinedHTTPPort).
 			WithStatusCodeMatcher(func(status int) bool { return status == 200 }).
-			WithStartupTimeout(90 * time.Second),
+			WithStartupTimeout(120 * time.Second),
 	}
 
 	c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
@@ -95,50 +124,109 @@ func StartCombined(ctx context.Context) (*Combined, error) {
 		Started:          true,
 	})
 	if err != nil {
+		_ = net.Remove(ctx)
 		return nil, fmt.Errorf("start combined container: %w", err)
 	}
 
 	host, err := c.Host(ctx)
 	if err != nil {
 		_ = c.Terminate(ctx)
+		_ = net.Remove(ctx)
 		return nil, fmt.Errorf("container host: %w", err)
 	}
 	mapped, err := c.MappedPort(ctx, nat.Port(combinedHTTPPort))
 	if err != nil {
 		_ = c.Terminate(ctx)
+		_ = net.Remove(ctx)
 		return nil, fmt.Errorf("mapped port: %w", err)
 	}
 
 	return &Combined{
 		container: c,
+		network:   net,
 		BaseURL:   fmt.Sprintf("http://%s:%s", host, mapped.Port()),
 		workDir:   workDir,
 	}, nil
 }
 
-// buildCombinedImage builds the combined server image via the docker CLI with
+// resolveImage returns the image to run for a component. The env override (or
-// BuildKit enabled, so the Dockerfile's cache mounts work and unchanged sources
+// the default) is used as-is when it looks like a registry reference (contains
-// reuse the layer + go caches. The image is tagged stably so reruns are cheap.
+// "/") — testcontainers pulls it. A bare local tag is built from the repo
-func buildCombinedImage(ctx context.Context, root string) error {
+// Dockerfile instead, so developers can still test source builds.
+func resolveImage(ctx context.Context, root, envKey, defaultImage, dockerfile string) (string, error) {
+	img := defaultImage
+	if v := os.Getenv(envKey); v != "" {
+		img = v
+	}
+	if strings.Contains(img, "/") {
+		return img, nil
+	}
+	if err := buildImage(ctx, root, dockerfile, img); err != nil {
+		return "", err
+	}
+	return img, nil
+}
+
+// buildImage builds an image from a repo Dockerfile via the docker CLI with
+// BuildKit enabled, so cache mounts work and unchanged sources reuse the layer
+// + go caches. Tags are stable so reruns are cheap.
+func buildImage(ctx context.Context, root, dockerfile, tag string) error {
 	cmd := exec.CommandContext(ctx, "docker", "build",
-		"-f", combinedDockerfile,
+		"-f", dockerfile,
-		"-t", combinedImageTag,
+		"-t", tag,
 		".",
 	)
 	cmd.Dir = root
 	cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1")
 	if out, err := cmd.CombinedOutput(); err != nil {
-		return fmt.Errorf("build combined image: %w\n%s", err, string(out))
+		return fmt.Errorf("build image %s: %w\n%s", tag, err, string(out))
 	}
 	return nil
 }
 
-// Terminate stops the container and removes the work dir.
+// CreateProxyTokenCLI mints a proxy access token via the server's `token
+// create` CLI inside the container — the same path the manual install uses.
+// This yields a GLOBAL (account-less) token, so the proxy serves the whole
+// cluster (SynthesizeServicesForCluster); an account-scoped REST token instead
+// drives the per-account path. Returns the plaintext token.
+func (c *Combined) CreateProxyTokenCLI(ctx context.Context, name string) (string, error) {
+	code, reader, err := c.container.Exec(ctx,
+		[]string{"/go/bin/netbird-server", "token", "create", "--name", name, "--config", "/nb/config.yaml"},
+		tcexec.Multiplexed())
+	if err != nil {
+		return "", fmt.Errorf("exec token create: %w", err)
+	}
+	out, _ := io.ReadAll(reader)
+	if code != 0 {
+		return "", fmt.Errorf("token create exited %d: %s", code, string(out))
+	}
+	for _, line := range strings.Split(string(out), "\n") {
+		line = strings.TrimSpace(line)
+		if strings.HasPrefix(line, "Token:") {
+			tok := strings.TrimSpace(strings.TrimPrefix(line, "Token:"))
+			if tok != "" {
+				return tok, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("token not found in CLI output: %s", string(out))
+}
+
+// Logs returns the combined server container logs, for diagnostics.
+func (c *Combined) Logs(ctx context.Context) string {
+	return containerLogs(ctx, c.container)
+}
+
+// Terminate stops the container, removes the shared network, and cleans the
+// work dir.
 func (c *Combined) Terminate(ctx context.Context) error {
 	var err error
 	if c.container != nil {
 		err = c.container.Terminate(ctx)
 	}
+	if c.network != nil {
+		_ = c.network.Remove(ctx)
+	}
 	if c.workDir != "" {
 		_ = os.RemoveAll(c.workDir)
 	}

e2e/harness/config.go

@@ -17,6 +17,8 @@ const combinedConfigYAML = `server:
   logFile: "console"
   authSecret: "e2e-relay-secret"
   dataDir: "/nb/data"
+  disableAnonymousMetrics: true
+  disableGeoliteUpdate: true
   auth:
     issuer: "%s"
   store:

e2e/harness/proxy.go

@@ -0,0 +1,116 @@
+//go:build e2e
+
+package harness
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/wait"
+)
+
+const (
+	proxyDockerfile = "proxy/Dockerfile.multistage"
+	// defaultProxyImage is the published reverse-proxy release used by default.
+	// Override with NB_E2E_PROXY_IMAGE; a value without a "/" is built locally.
+	defaultProxyImage = "netbirdio/reverse-proxy:0.74.0-rc.2"
+	proxyAlias        = "proxy"
+
+	// AgentNetworkCluster is the proxy cluster the e2e provider bootstraps and
+	// the proxy serves. It must equal the management's exposed domain
+	// (combinedAlias) — the working manual setup uses one NETBIRD_DOMAIN for
+	// both. The agent-network endpoint is <subdomain>.<cluster>.
+	AgentNetworkCluster = combinedAlias
+)
+
+// Proxy is a running agent-network gateway (netbird proxy) container.
+type Proxy struct {
+	container testcontainers.Container
+	workDir   string
+}
+
+// StartProxy builds the proxy image and runs it on the combined server's
+// network, registered via the given account proxy token and serving the
+// AgentNetworkCluster over a self-signed wildcard cert. It does not wait for
+// peer connectivity — callers poll management for the proxy peer.
+func StartProxy(ctx context.Context, c *Combined, proxyToken string) (*Proxy, error) {
+	root, err := repoRoot()
+	if err != nil {
+		return nil, err
+	}
+	proxyImage, err := resolveImage(ctx, root, "NB_E2E_PROXY_IMAGE", defaultProxyImage, proxyDockerfile)
+	if err != nil {
+		return nil, err
+	}
+
+	workDir, err := os.MkdirTemp("/tmp", "nb-e2e-proxy-*")
+	if err != nil {
+		return nil, fmt.Errorf("create proxy work dir: %w", err)
+	}
+	if err := writeSelfSignedCert(workDir, []string{"*." + AgentNetworkCluster, AgentNetworkCluster}); err != nil {
+		return nil, err
+	}
+
+	req := testcontainers.ContainerRequest{
+		Image:          proxyImage,
+		Networks:       []string{c.network.Name},
+		NetworkAliases: map[string][]string{c.network.Name: {proxyAlias}},
+		Env: map[string]string{
+			"NB_PROXY_TOKEN":                 proxyToken,
+			"NB_PROXY_MANAGEMENT_ADDRESS":    combinedExposedURL,
+			"NB_PROXY_DOMAIN":                AgentNetworkCluster,
+			"NB_PROXY_ADDRESS":               ":443",
+			"NB_PROXY_CERTIFICATE_DIRECTORY": "/certs",
+			"NB_PROXY_HEALTH_ADDRESS":        ":8081",
+			"NB_PROXY_LOG_LEVEL":             "debug",
+			"NB_PROXY_PRIVATE":               "true",
+			// Management is plain HTTP in-cluster, so allow the proxy token to
+			// ride a non-TLS gRPC connection.
+			"NB_PROXY_ALLOW_INSECURE": "true",
+			// The combined server multiplexes the relay over WebSocket on :8080
+			// (no QUIC listener). The proxy's embedded relay client defaults to
+			// QUIC, which fails here and flaps the relay link, churning the
+			// proxy peer so it never stably registers. Force WS transport.
+			"NB_RELAY_TRANSPORT": "ws",
+			// Trace the embedded client (relay / signal / handshake) so
+			// peer-registration issues are visible in the proxy logs.
+			"NB_PROXY_CLIENT_LOG_LEVEL": "trace",
+		},
+		HostConfigModifier: func(hc *container.HostConfig) {
+			hc.Binds = append(hc.Binds, workDir+":/certs")
+			hc.CapAdd = append(hc.CapAdd, "NET_ADMIN", "SYS_ADMIN", "SYS_RESOURCE", "NET_BIND_SERVICE")
+		},
+		WaitingFor: wait.ForLog("Initial mapping sync complete").WithStartupTimeout(90 * time.Second),
+	}
+
+	ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: req,
+		Started:          true,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("start proxy container: %w", err)
+	}
+
+	return &Proxy{container: ctr, workDir: workDir}, nil
+}
+
+// Logs returns the proxy container logs, for diagnostics on failure.
+func (p *Proxy) Logs(ctx context.Context) string {
+	return containerLogs(ctx, p.container)
+}
+
+// Terminate stops the proxy container and cleans its work dir.
+func (p *Proxy) Terminate(ctx context.Context) error {
+	var err error
+	if p.container != nil {
+		err = p.container.Terminate(ctx)
+	}
+	if p.workDir != "" {
+		_ = os.RemoveAll(p.workDir)
+	}
+	return err
+}