From 3e6aba7dfe83a9978bfdd0aca7c3aea507fc5986 Mon Sep 17 00:00:00 2001 From: mlsmaycon Date: Mon, 29 Jun 2026 10:34:43 +0200 Subject: [PATCH] [e2e] Add live chat-through-proxy scenario (Pillar 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stand up the full agent-network data path in containers and drive a real chat-completion through the gateway: - harness: a shared docker network (combined server reachable by alias), a proxy container built from the published reverse-proxy image (NB_PROXY_PRIVATE, NB_PROXY_ALLOW_INSECURE, NB_RELAY_TRANSPORT=ws to match the combined server's WS-multiplexed relay) with a generated self-signed wildcard cert, and a netbird client container that joins via a setup key. - the combined image, proxy image, and client image default to the published rc.2 releases (overridable via NB_E2E_*_IMAGE; a bare local tag is built from source instead). Geolocation download is disabled so the server starts without external fetches. - one shared domain is used for the management exposed address, the proxy domain, and the agent-network cluster; the proxy token is minted via the server CLI (global) to match the manual install. TestChatCompletionThroughProxy provisions provider+policy+group+setup key, runs proxy+client, drives an OpenAI chat-completion through the tunnel, and asserts a 200 plus the ingested access-log row. Requires OPENAI_TOKEN (skips otherwise). The provider must be created with enabled=true explicitly — the create default is false despite the API doc. --- e2e/agentnetwork/chat_test.go | 165 +++++++++++++++++++++++++++ e2e/harness/Dockerfile.client | 24 ++++ e2e/harness/agentnetwork.go | 11 ++ e2e/harness/bootstrap.go | 2 +- e2e/harness/cert.go | 64 +++++++++++ e2e/harness/client.go | 207 ++++++++++++++++++++++++++++++++++ e2e/harness/combined.go | 142 ++++++++++++++++++----- e2e/harness/config.go | 2 + e2e/harness/proxy.go | 116 +++++++++++++++++++ 9 files changed, 705 insertions(+), 28 deletions(-) create mode 100644 e2e/agentnetwork/chat_test.go create mode 100644 e2e/harness/Dockerfile.client create mode 100644 e2e/harness/cert.go create mode 100644 e2e/harness/client.go create mode 100644 e2e/harness/proxy.go diff --git a/e2e/agentnetwork/chat_test.go b/e2e/agentnetwork/chat_test.go new file mode 100644 index 000000000..a2e27f4e3 --- /dev/null +++ b/e2e/agentnetwork/chat_test.go @@ -0,0 +1,165 @@ +//go:build e2e + +package agentnetwork + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/e2e/harness" + "github.com/netbirdio/netbird/shared/management/http/api" +) + +// TestChatCompletionThroughProxy is Pillar 3: it provisions an agent-network +// gateway (provider + policy + setup key), runs the proxy and a client +// container on the shared network, and drives a real chat-completion from the +// client through the proxy to the upstream provider over the WireGuard tunnel, +// asserting a 200 and that usage is recorded. +// +// Requires a real provider key in OPENAI_TOKEN (source ~/.llm-keys locally; set +// the Actions secret in CI). Skips otherwise. +func TestChatCompletionThroughProxy(t *testing.T) { + apiKey := os.Getenv("OPENAI_TOKEN") + if apiKey == "" { + t.Skip("OPENAI_TOKEN not set; source ~/.llm-keys to run the live chat test") + } + + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) + defer cancel() + + // Group + setup key: the client joins into this group; the policy authorizes + // it to reach the provider. + grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-agents"}) + require.NoError(t, err, "create agents group") + t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) }) + + ephemeral := false + sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{ + Name: "e2e-client", + Type: "reusable", + ExpiresIn: 86400, + UsageLimit: 0, + AutoGroups: []string{grp.Id}, + Ephemeral: &ephemeral, + }) + require.NoError(t, err, "mint setup key") + require.NotEmpty(t, sk.Key, "setup key must be returned in plaintext") + + // Provider (real upstream key) + policy authorizing the group. Created + // before the proxy starts so the proxy's initial cluster snapshot already + // carries the account's synthesized service. + prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ + Name: "OpenAI Live", + ProviderId: "openai_api", + UpstreamUrl: "https://api.openai.com", + ApiKey: &apiKey, + Enabled: ptr(true), + BootstrapCluster: ptr(harness.AgentNetworkCluster), + Models: &[]api.AgentNetworkProviderModel{ + {Id: "gpt-4o-mini", InputPer1k: 0.00015, OutputPer1k: 0.0006}, + }, + }) + require.NoError(t, err, "create provider") + t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) + + enabled := true + policyReq := api.AgentNetworkPolicyRequest{ + Name: "e2e-allow", + Enabled: &enabled, + SourceGroups: []string{grp.Id}, + DestinationProviderIds: []string{prov.Id}, + } + pol, err := srv.CreatePolicy(ctx, policyReq) + require.NoError(t, err, "create policy") + t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) }) + + settings, err := srv.GetSettings(ctx) + require.NoError(t, err, "read settings for endpoint") + require.NotEmpty(t, settings.Endpoint, "agent-network endpoint must be assigned") + + // Mint the proxy token via the server CLI (global, account-less) — the path + // the manual install uses, which drives the cluster-snapshot synthesis the + // proxy needs. An account-scoped REST token takes a different path that + // doesn't deliver the service. + proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-proxy") + require.NoError(t, err, "mint proxy token via CLI") + + px, err := harness.StartProxy(ctx, srv, proxyToken) + require.NoError(t, err, "start proxy") + t.Cleanup(func() { _ = px.Terminate(context.Background()) }) + + // Client joins last, once the proxy + provider + policy are all in place, so + // its initial network map includes the synthesized agent-network service. + cl, err := harness.StartClient(ctx, srv, sk.Key) + require.NoError(t, err, "start client") + t.Cleanup(func() { _ = cl.Terminate(context.Background()) }) + + require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management") + + if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil { + dctx := context.Background() + peers, _ := srv.API().Peers.List(dctx) + var peerInfo []string + for _, p := range peers { + var groups []string + for _, g := range p.Groups { + groups = append(groups, g.Name) + } + peerInfo = append(peerInfo, fmt.Sprintf("%s connected=%t ip=%s groups=%v", p.Name, p.Connected, p.Ip, groups)) + } + clusters, _ := srv.API().ReverseProxyClusters.List(dctx) + var clusterInfo []string + for _, cl := range clusters { + clusterInfo = append(clusterInfo, fmt.Sprintf("%+v", cl)) + } + domains, _ := srv.API().ReverseProxyDomains.List(dctx) + var domainInfo []string + for _, d := range domains { + domainInfo = append(domainInfo, fmt.Sprintf("%+v", d)) + } + _ = os.WriteFile("/tmp/nb-e2e-proxy.log", []byte(px.Logs(dctx)), 0o644) + _ = os.WriteFile("/tmp/nb-e2e-client.log", []byte(cl.Logs(dctx)), 0o644) + _ = os.WriteFile("/tmp/nb-e2e-combined.log", []byte(srv.Logs(dctx)), 0o644) + diag := fmt.Sprintf("settings: cluster=%q endpoint=%q subdomain=%q\nprovider: id=%s cluster=%s\npolicy: id=%s sourceGroups=%v dst=%v\ngroup: id=%s\npeers:\n%s\nclusters:\n%s\n", + settings.Cluster, settings.Endpoint, settings.Subdomain, + prov.Id, harness.AgentNetworkCluster, + pol.Id, policyReq.SourceGroups, policyReq.DestinationProviderIds, + grp.Id, + strings.Join(peerInfo, "\n"), strings.Join(clusterInfo, "\n")) + diag += "domains:\n" + strings.Join(domainInfo, "\n") + "\n" + _ = os.WriteFile("/tmp/nb-e2e-diag.txt", []byte(diag), 0o644) + t.Fatalf("client did not see the proxy peer: %v\n=== settings ===\ncluster=%q endpoint=%q subdomain=%q\n=== peers ===\n%v\n=== clusters ===\n%v\n=== proxy logs ===\n%s", + err, settings.Cluster, settings.Endpoint, settings.Subdomain, peerInfo, clusterInfo, px.Logs(dctx)) + } + + proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint) + require.NoError(t, err, "resolve agent-network endpoint to proxy IP") + + code, body, err := cl.Chat(ctx, settings.Endpoint, proxyIP, "gpt-4o-mini", "Reply with exactly: pong") + require.NoError(t, err, "chat request through tunnel") + if code != 200 { + t.Fatalf("expected 200 from chat-completion, got %d\nbody: %s\n=== proxy logs ===\n%s", code, body, px.Logs(context.Background())) + } + assert.Contains(t, body, "choices", "chat response should carry choices") + + // The per-request access-log row is ingested asynchronously after the + // response is forwarded; poll briefly. (Consumption rows are only booked + // when a policy has token/budget limits, which this one doesn't.) + require.Eventually(t, func() bool { + resp, lerr := srv.ListAccessLogs(ctx) + return lerr == nil && resp.TotalRecords > 0 + }, 30*time.Second, 2*time.Second, "an access-log row should be recorded after the chat-completion") + + logs, err := srv.ListAccessLogs(ctx) + require.NoError(t, err, "read access logs") + require.NotEmpty(t, logs.Data, "access-log page must contain the request row") + require.NotNil(t, logs.Data[0].Model, "access-log row should record the model") + assert.Equal(t, "gpt-4o-mini", *logs.Data[0].Model, "access-log row should record the requested model") +} diff --git a/e2e/harness/Dockerfile.client b/e2e/harness/Dockerfile.client new file mode 100644 index 000000000..114577d60 --- /dev/null +++ b/e2e/harness/Dockerfile.client @@ -0,0 +1,24 @@ +# Multistage build for the NetBird client used in e2e tests. The repo has no +# source-building client Dockerfile (client/Dockerfile packages a goreleaser +# artifact), so this mirrors its alpine runtime + entrypoint while compiling the +# CGO-free client inline. BuildKit cache mounts keep rebuilds incremental. + +FROM golang:1.25-bookworm AS builder +WORKDIR /src +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download +COPY . . +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=linux go build -o /out/netbird ./client + +FROM alpine:3.24 +RUN apk add --no-cache bash ca-certificates ip6tables iproute2 iptables +ENV NETBIRD_BIN="/usr/local/bin/netbird" \ + NB_LOG_FILE="console,/var/log/netbird/client.log" \ + NB_DAEMON_ADDR="unix:///var/run/netbird.sock" \ + NB_ENABLE_CAPTURE="false" \ + NB_ENTRYPOINT_SERVICE_TIMEOUT="30" +ENTRYPOINT [ "/usr/local/bin/netbird-entrypoint.sh" ] +COPY client/netbird-entrypoint.sh /usr/local/bin/netbird-entrypoint.sh +COPY --from=builder /out/netbird /usr/local/bin/netbird diff --git a/e2e/harness/agentnetwork.go b/e2e/harness/agentnetwork.go index 03cfbae5b..a21fd1cf3 100644 --- a/e2e/harness/agentnetwork.go +++ b/e2e/harness/agentnetwork.go @@ -79,6 +79,11 @@ func (c *Combined) CreatePolicy(ctx context.Context, req api.AgentNetworkPolicyR return anRequest[api.AgentNetworkPolicy](ctx, c, http.MethodPost, "/api/agent-network/policies", req) } +// UpdatePolicy replaces a policy by id. +func (c *Combined) UpdatePolicy(ctx context.Context, id string, req api.AgentNetworkPolicyRequest) (api.AgentNetworkPolicy, error) { + return anRequest[api.AgentNetworkPolicy](ctx, c, http.MethodPut, "/api/agent-network/policies/"+id, req) +} + // DeletePolicy removes a policy by id. func (c *Combined) DeletePolicy(ctx context.Context, id string) error { return anDelete(ctx, c, "/api/agent-network/policies/"+id) @@ -99,3 +104,9 @@ func (c *Combined) UpdateSettings(ctx context.Context, req api.AgentNetworkSetti func (c *Combined) ListConsumption(ctx context.Context) ([]api.AgentNetworkConsumption, error) { return anRequest[[]api.AgentNetworkConsumption](ctx, c, http.MethodGet, "/api/agent-network/consumption", nil) } + +// ListAccessLogs returns the account's agent-network access-log page (the +// flattened per-request rows the proxy ships and management ingests). +func (c *Combined) ListAccessLogs(ctx context.Context) (api.AgentNetworkAccessLogsResponse, error) { + return anRequest[api.AgentNetworkAccessLogsResponse](ctx, c, http.MethodGet, "/api/agent-network/access-logs", nil) +} diff --git a/e2e/harness/bootstrap.go b/e2e/harness/bootstrap.go index bc44528f8..defa03c14 100644 --- a/e2e/harness/bootstrap.go +++ b/e2e/harness/bootstrap.go @@ -21,7 +21,7 @@ func (c *Combined) Bootstrap(ctx context.Context) (string, error) { createPAT := true expireDays := 1 - resp, err := setupClient.Instance.Setup(ctx, api.PostApiSetupJSONRequestBody{ + resp, err := setupClient.Instance.Setup(ctx, api.PostApiSetupJSONRequestBody{ //nolint:gosec // static throwaway test credentials Email: "admin@netbird.test", Password: "Netbird-e2e-Passw0rd!", Name: "E2E Admin", diff --git a/e2e/harness/cert.go b/e2e/harness/cert.go new file mode 100644 index 000000000..de898e7e0 --- /dev/null +++ b/e2e/harness/cert.go @@ -0,0 +1,64 @@ +//go:build e2e + +package harness + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "os" + "path/filepath" + "time" +) + +// writeSelfSignedCert generates a self-signed TLS cert/key pair covering the +// given DNS names and writes them as tls.crt / tls.key in dir. The proxy serves +// this for the agent-network endpoint; the client curls with -k, so validity +// chains don't matter — the proxy just needs a usable cert to present. +func writeSelfSignedCert(dir string, dnsNames []string) error { + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return fmt.Errorf("generate key: %w", err) + } + + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return fmt.Errorf("generate serial: %w", err) + } + + tmpl := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: dnsNames[0]}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: dnsNames, + BasicConstraintsValid: true, + } + + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv) + if err != nil { + return fmt.Errorf("create certificate: %w", err) + } + + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + if err := os.WriteFile(filepath.Join(dir, "tls.crt"), certPEM, 0o644); err != nil { //nolint:gosec // public cert, bind-mounted and read by the proxy container + return fmt.Errorf("write cert: %w", err) + } + + keyDER, err := x509.MarshalECPrivateKey(priv) + if err != nil { + return fmt.Errorf("marshal key: %w", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + if err := os.WriteFile(filepath.Join(dir, "tls.key"), keyPEM, 0o600); err != nil { + return fmt.Errorf("write key: %w", err) + } + return nil +} diff --git a/e2e/harness/client.go b/e2e/harness/client.go new file mode 100644 index 000000000..1b94a23a5 --- /dev/null +++ b/e2e/harness/client.go @@ -0,0 +1,207 @@ +//go:build e2e + +package harness + +import ( + "context" + "fmt" + "io" + "os/exec" + "strings" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/testcontainers/testcontainers-go" + tcexec "github.com/testcontainers/testcontainers-go/exec" +) + +const ( + clientDockerfile = "e2e/harness/Dockerfile.client" + // defaultClientImage is the published NetBird client release used by + // default. Override with NB_E2E_CLIENT_IMAGE; a value without a "/" is built + // locally from clientDockerfile. + defaultClientImage = "netbirdio/netbird:0.74.0-rc.2" + clientAlias = "client" + curlImage = "curlimages/curl:latest" +) + +// Client is a running NetBird client container joined to the combined server. +type Client struct { + container testcontainers.Container +} + +// StartClient builds the client image and runs it on the combined server's +// network, joining via the given setup key. The image entrypoint brings the +// daemon up automatically; callers wait for connectivity with WaitConnected / +// WaitProxyPeer. +func StartClient(ctx context.Context, c *Combined, setupKey string) (*Client, error) { + root, err := repoRoot() + if err != nil { + return nil, err + } + clientImage, err := resolveImage(ctx, root, "NB_E2E_CLIENT_IMAGE", defaultClientImage, clientDockerfile) + if err != nil { + return nil, err + } + + req := testcontainers.ContainerRequest{ + Image: clientImage, + Networks: []string{c.network.Name}, + NetworkAliases: map[string][]string{c.network.Name: {clientAlias}}, + Env: map[string]string{ + "NB_MANAGEMENT_URL": combinedExposedURL, + "NB_SETUP_KEY": setupKey, + "NB_LOG_LEVEL": "info", + // Match the proxy: the combined relay is WebSocket-only, so the + // client must use WS transport to keep a stable relay link to it. + "NB_RELAY_TRANSPORT": "ws", + }, + HostConfigModifier: func(hc *container.HostConfig) { + hc.CapAdd = append(hc.CapAdd, "NET_ADMIN", "SYS_ADMIN", "SYS_RESOURCE") + }, + } + + ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + return nil, fmt.Errorf("start client container: %w", err) + } + return &Client{container: ctr}, nil +} + +// Restart bounces the client connection (netbird down/up) so it pulls a fresh +// network map — the documented workaround for a freshly-joined client not yet +// seeing a synthesized agent-network service. +func (cl *Client) Restart(ctx context.Context) error { + if _, _, err := cl.container.Exec(ctx, []string{"netbird", "down"}, tcexec.Multiplexed()); err != nil { + return fmt.Errorf("netbird down: %w", err) + } + time.Sleep(2 * time.Second) + code, reader, err := cl.container.Exec(ctx, []string{"netbird", "up"}, tcexec.Multiplexed()) + if err != nil { + return fmt.Errorf("netbird up: %w", err) + } + if code != 0 { + out, _ := io.ReadAll(reader) + return fmt.Errorf("netbird up exited %d: %s", code, string(out)) + } + return nil +} + +// Status returns `netbird status` output from inside the client. +func (cl *Client) Status(ctx context.Context) (string, error) { + code, reader, err := cl.container.Exec(ctx, []string{"netbird", "status"}, tcexec.Multiplexed()) + if err != nil { + return "", err + } + out, _ := io.ReadAll(reader) + if code != 0 { + return string(out), fmt.Errorf("netbird status exited %d", code) + } + return string(out), nil +} + +// WaitConnected polls until the client reports Management: Connected. +func (cl *Client) WaitConnected(ctx context.Context, timeout time.Duration) error { + return cl.pollStatus(ctx, timeout, "Management: Connected") +} + +// WaitProxyPeer polls until the client sees the proxy peer connected (1/1). +func (cl *Client) WaitProxyPeer(ctx context.Context, timeout time.Duration) error { + return cl.pollStatus(ctx, timeout, "1/1 Connected") +} + +func (cl *Client) pollStatus(ctx context.Context, timeout time.Duration, want string) error { + deadline := time.Now().Add(timeout) + var last string + for time.Now().Before(deadline) { + out, _ := cl.Status(ctx) + last = out + if strings.Contains(out, want) { + return nil + } + time.Sleep(3 * time.Second) + } + return fmt.Errorf("timed out waiting for %q; last status:\n%s", want, last) +} + +// ResolveProxyIP resolves the agent-network endpoint to the proxy peer's +// NetBird IP from inside the client (via magic DNS). +func (cl *Client) ResolveProxyIP(ctx context.Context, endpoint string) (string, error) { + code, reader, err := cl.container.Exec(ctx, []string{"getent", "hosts", endpoint}, tcexec.Multiplexed()) + if err != nil { + return "", err + } + out, _ := io.ReadAll(reader) + if code != 0 { + return "", fmt.Errorf("getent hosts %s exited %d", endpoint, code) + } + fields := strings.Fields(string(out)) + if len(fields) == 0 { + return "", fmt.Errorf("no address for %s", endpoint) + } + return fields[0], nil +} + +// Chat issues a chat-completion POST to the agent-network endpoint over the +// client's tunnel, returning the HTTP status and response body. It runs curl in +// a throwaway container sharing the client's network namespace so the request +// traverses the WireGuard tunnel, pinning the endpoint to the proxy peer IP. +func (cl *Client) Chat(ctx context.Context, endpoint, proxyIP, model, prompt string) (int, string, error) { + body := fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":%q}]}`, model, prompt) + url := "https://" + endpoint + "/v1/chat/completions" + + args := []string{ + "run", "--rm", + "--network", "container:" + cl.container.GetContainerID(), + curlImage, + "-sk", "--connect-timeout", "5", "--max-time", "90", + "--resolve", endpoint + ":443:" + proxyIP, + "-o", "/dev/stderr", "-w", "%{http_code}", + "-X", "POST", url, + "-H", "Content-Type: application/json", + "--data", body, + } + cmd := exec.CommandContext(ctx, "docker", args...) + // -w writes the status code to stdout; -o /dev/stderr writes the body to + // stderr so we can capture both separately. + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return 0, stderr.String(), fmt.Errorf("curl through tunnel: %w", err) + } + + code := 0 + _, _ = fmt.Sscanf(strings.TrimSpace(stdout.String()), "%d", &code) + return code, stderr.String(), nil +} + +// Logs returns the client container logs, for diagnostics on failure. +func (cl *Client) Logs(ctx context.Context) string { + return containerLogs(ctx, cl.container) +} + +// Terminate stops the client container. +func (cl *Client) Terminate(ctx context.Context) error { + if cl.container == nil { + return nil + } + return cl.container.Terminate(ctx) +} + +// containerLogs reads up to 256 KiB of a container's logs for diagnostics. +func containerLogs(ctx context.Context, c testcontainers.Container) string { + if c == nil { + return "" + } + r, err := c.Logs(ctx) + if err != nil { + return fmt.Sprintf("", err) + } + defer r.Close() + b, _ := io.ReadAll(io.LimitReader(r, 256<<10)) + return string(b) +} diff --git a/e2e/harness/combined.go b/e2e/harness/combined.go index 9a9633e16..0f948ecf6 100644 --- a/e2e/harness/combined.go +++ b/e2e/harness/combined.go @@ -5,14 +5,18 @@ package harness import ( "context" "fmt" + "io" "os" "os/exec" "path/filepath" + "strings" "time" "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" "github.com/testcontainers/testcontainers-go" + tcexec "github.com/testcontainers/testcontainers-go/exec" + "github.com/testcontainers/testcontainers-go/network" "github.com/testcontainers/testcontainers-go/wait" "github.com/netbirdio/netbird/shared/management/client/rest" @@ -20,20 +24,34 @@ import ( const ( combinedDockerfile = "combined/Dockerfile.multistage" - combinedImageTag = "netbird-e2e-combined:latest" - combinedHTTPPort = "8080/tcp" + // defaultCombinedImage is the published combined-server release used by + // default (the artifact operators run manually). Override with + // NB_E2E_COMBINED_IMAGE; a value without a "/" is built locally from + // combinedDockerfile instead of pulled. + defaultCombinedImage = "netbirdio/netbird-server:0.74.0-rc.2" + combinedHTTPPort = "8080/tcp" - // containerInternalURL is how the server addresses itself: it listens on - // :8080 inside the container, so the embedded IdP issuer and exposed - // address both resolve there. Tests reach it via the mapped host port. - containerInternalURL = "http://localhost:8080" - containerIssuer = containerInternalURL + "/oauth2" + // combinedAlias is the combined server's network alias AND the deployment + // domain. The working manual setup uses a single NETBIRD_DOMAIN for the + // management exposed address, the proxy domain, and the agent-network + // cluster — so we mirror that: peers reach management/signal/relay at this + // name, the proxy registers this as its cluster, and the agent-network + // endpoint is .. + combinedAlias = "netbird.local" + combinedExposedURL = "http://" + combinedAlias + ":8080" + + // containerIssuer is the embedded IdP issuer, used only for internal JWT + // validation (peers authenticate with setup keys / proxy tokens, not OIDC), + // so the in-container localhost address is fine. + containerIssuer = "http://localhost:8080/oauth2" ) // Combined is a running combined NetBird server (management + signal + relay + -// STUN + embedded IdP) plus the connection details tests need. +// STUN + embedded IdP) plus the connection details tests need. It owns the +// shared docker network that the proxy and client containers join. type Combined struct { container testcontainers.Container + network *testcontainers.DockerNetwork // BaseURL is the host-reachable management API root, e.g. http://127.0.0.1:51234. BaseURL string // PAT is the admin Personal Access Token minted via Bootstrap. @@ -44,41 +62,52 @@ type Combined struct { } // StartCombined builds the combined server from its multistage Dockerfile and -// boots it with setup-PAT enabled, returning once the API is serving. The -// caller still owns minting the admin PAT via Bootstrap. +// boots it with setup-PAT enabled on a fresh shared network, returning once the +// API is serving. The caller still owns minting the admin PAT via Bootstrap. func StartCombined(ctx context.Context) (*Combined, error) { root, err := repoRoot() if err != nil { return nil, err } - // Build with the docker CLI (BuildKit) rather than testcontainers' classic - // build path so the Dockerfile's cache mounts are honored and recompiles - // stay incremental. testcontainers then just runs the tagged image. - if err := buildCombinedImage(ctx, root); err != nil { + combinedImage, err := resolveImage(ctx, root, "NB_E2E_COMBINED_IMAGE", defaultCombinedImage, combinedDockerfile) + if err != nil { return nil, err } + net, err := network.New(ctx) + if err != nil { + return nil, fmt.Errorf("create shared network: %w", err) + } + // Work dir under /tmp so Docker Desktop file sharing (which excludes // macOS's /var/folders TMPDIR) can bind-mount it. workDir, err := os.MkdirTemp("/tmp", "nb-e2e-combined-*") if err != nil { + _ = net.Remove(ctx) return nil, fmt.Errorf("create work dir: %w", err) } - cfg := fmt.Sprintf(combinedConfigYAML, containerInternalURL, containerIssuer) - if err := os.WriteFile(filepath.Join(workDir, "config.yaml"), []byte(cfg), 0o644); err != nil { + cfg := fmt.Sprintf(combinedConfigYAML, combinedExposedURL, containerIssuer) + if err := os.WriteFile(filepath.Join(workDir, "config.yaml"), []byte(cfg), 0o644); err != nil { //nolint:gosec // non-secret config, bind-mounted and read by the container + _ = net.Remove(ctx) return nil, fmt.Errorf("write combined config: %w", err) } if err := os.MkdirAll(filepath.Join(workDir, "data"), 0o755); err != nil { + _ = net.Remove(ctx) return nil, fmt.Errorf("create datadir: %w", err) } req := testcontainers.ContainerRequest{ - Image: combinedImageTag, - ExposedPorts: []string{combinedHTTPPort}, + Image: combinedImage, + ExposedPorts: []string{combinedHTTPPort}, + Networks: []string{net.Name}, + NetworkAliases: map[string][]string{net.Name: {combinedAlias}}, Env: map[string]string{ "NB_SETUP_PAT_ENABLED": "true", + // Skip the GeoLite DB download — it blocks startup and agent-network + // ingest doesn't use geolocation. + "NB_DISABLE_GEOLOCATION": "true", }, Cmd: []string{"--config", "/nb/config.yaml"}, HostConfigModifier: func(hc *container.HostConfig) { @@ -87,7 +116,7 @@ func StartCombined(ctx context.Context) (*Combined, error) { WaitingFor: wait.ForHTTP("/api/instance"). WithPort(combinedHTTPPort). WithStatusCodeMatcher(func(status int) bool { return status == 200 }). - WithStartupTimeout(90 * time.Second), + WithStartupTimeout(120 * time.Second), } c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ @@ -95,50 +124,109 @@ func StartCombined(ctx context.Context) (*Combined, error) { Started: true, }) if err != nil { + _ = net.Remove(ctx) return nil, fmt.Errorf("start combined container: %w", err) } host, err := c.Host(ctx) if err != nil { _ = c.Terminate(ctx) + _ = net.Remove(ctx) return nil, fmt.Errorf("container host: %w", err) } mapped, err := c.MappedPort(ctx, nat.Port(combinedHTTPPort)) if err != nil { _ = c.Terminate(ctx) + _ = net.Remove(ctx) return nil, fmt.Errorf("mapped port: %w", err) } return &Combined{ container: c, + network: net, BaseURL: fmt.Sprintf("http://%s:%s", host, mapped.Port()), workDir: workDir, }, nil } -// buildCombinedImage builds the combined server image via the docker CLI with -// BuildKit enabled, so the Dockerfile's cache mounts work and unchanged sources -// reuse the layer + go caches. The image is tagged stably so reruns are cheap. -func buildCombinedImage(ctx context.Context, root string) error { +// resolveImage returns the image to run for a component. The env override (or +// the default) is used as-is when it looks like a registry reference (contains +// "/") — testcontainers pulls it. A bare local tag is built from the repo +// Dockerfile instead, so developers can still test source builds. +func resolveImage(ctx context.Context, root, envKey, defaultImage, dockerfile string) (string, error) { + img := defaultImage + if v := os.Getenv(envKey); v != "" { + img = v + } + if strings.Contains(img, "/") { + return img, nil + } + if err := buildImage(ctx, root, dockerfile, img); err != nil { + return "", err + } + return img, nil +} + +// buildImage builds an image from a repo Dockerfile via the docker CLI with +// BuildKit enabled, so cache mounts work and unchanged sources reuse the layer +// + go caches. Tags are stable so reruns are cheap. +func buildImage(ctx context.Context, root, dockerfile, tag string) error { cmd := exec.CommandContext(ctx, "docker", "build", - "-f", combinedDockerfile, - "-t", combinedImageTag, + "-f", dockerfile, + "-t", tag, ".", ) cmd.Dir = root cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1") if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("build combined image: %w\n%s", err, string(out)) + return fmt.Errorf("build image %s: %w\n%s", tag, err, string(out)) } return nil } -// Terminate stops the container and removes the work dir. +// CreateProxyTokenCLI mints a proxy access token via the server's `token +// create` CLI inside the container — the same path the manual install uses. +// This yields a GLOBAL (account-less) token, so the proxy serves the whole +// cluster (SynthesizeServicesForCluster); an account-scoped REST token instead +// drives the per-account path. Returns the plaintext token. +func (c *Combined) CreateProxyTokenCLI(ctx context.Context, name string) (string, error) { + code, reader, err := c.container.Exec(ctx, + []string{"/go/bin/netbird-server", "token", "create", "--name", name, "--config", "/nb/config.yaml"}, + tcexec.Multiplexed()) + if err != nil { + return "", fmt.Errorf("exec token create: %w", err) + } + out, _ := io.ReadAll(reader) + if code != 0 { + return "", fmt.Errorf("token create exited %d: %s", code, string(out)) + } + for _, line := range strings.Split(string(out), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "Token:") { + tok := strings.TrimSpace(strings.TrimPrefix(line, "Token:")) + if tok != "" { + return tok, nil + } + } + } + return "", fmt.Errorf("token not found in CLI output: %s", string(out)) +} + +// Logs returns the combined server container logs, for diagnostics. +func (c *Combined) Logs(ctx context.Context) string { + return containerLogs(ctx, c.container) +} + +// Terminate stops the container, removes the shared network, and cleans the +// work dir. func (c *Combined) Terminate(ctx context.Context) error { var err error if c.container != nil { err = c.container.Terminate(ctx) } + if c.network != nil { + _ = c.network.Remove(ctx) + } if c.workDir != "" { _ = os.RemoveAll(c.workDir) } diff --git a/e2e/harness/config.go b/e2e/harness/config.go index 583e0d245..b4bed60a2 100644 --- a/e2e/harness/config.go +++ b/e2e/harness/config.go @@ -17,6 +17,8 @@ const combinedConfigYAML = `server: logFile: "console" authSecret: "e2e-relay-secret" dataDir: "/nb/data" + disableAnonymousMetrics: true + disableGeoliteUpdate: true auth: issuer: "%s" store: diff --git a/e2e/harness/proxy.go b/e2e/harness/proxy.go new file mode 100644 index 000000000..085ad6958 --- /dev/null +++ b/e2e/harness/proxy.go @@ -0,0 +1,116 @@ +//go:build e2e + +package harness + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +const ( + proxyDockerfile = "proxy/Dockerfile.multistage" + // defaultProxyImage is the published reverse-proxy release used by default. + // Override with NB_E2E_PROXY_IMAGE; a value without a "/" is built locally. + defaultProxyImage = "netbirdio/reverse-proxy:0.74.0-rc.2" + proxyAlias = "proxy" + + // AgentNetworkCluster is the proxy cluster the e2e provider bootstraps and + // the proxy serves. It must equal the management's exposed domain + // (combinedAlias) — the working manual setup uses one NETBIRD_DOMAIN for + // both. The agent-network endpoint is .. + AgentNetworkCluster = combinedAlias +) + +// Proxy is a running agent-network gateway (netbird proxy) container. +type Proxy struct { + container testcontainers.Container + workDir string +} + +// StartProxy builds the proxy image and runs it on the combined server's +// network, registered via the given account proxy token and serving the +// AgentNetworkCluster over a self-signed wildcard cert. It does not wait for +// peer connectivity — callers poll management for the proxy peer. +func StartProxy(ctx context.Context, c *Combined, proxyToken string) (*Proxy, error) { + root, err := repoRoot() + if err != nil { + return nil, err + } + proxyImage, err := resolveImage(ctx, root, "NB_E2E_PROXY_IMAGE", defaultProxyImage, proxyDockerfile) + if err != nil { + return nil, err + } + + workDir, err := os.MkdirTemp("/tmp", "nb-e2e-proxy-*") + if err != nil { + return nil, fmt.Errorf("create proxy work dir: %w", err) + } + if err := writeSelfSignedCert(workDir, []string{"*." + AgentNetworkCluster, AgentNetworkCluster}); err != nil { + return nil, err + } + + req := testcontainers.ContainerRequest{ + Image: proxyImage, + Networks: []string{c.network.Name}, + NetworkAliases: map[string][]string{c.network.Name: {proxyAlias}}, + Env: map[string]string{ + "NB_PROXY_TOKEN": proxyToken, + "NB_PROXY_MANAGEMENT_ADDRESS": combinedExposedURL, + "NB_PROXY_DOMAIN": AgentNetworkCluster, + "NB_PROXY_ADDRESS": ":443", + "NB_PROXY_CERTIFICATE_DIRECTORY": "/certs", + "NB_PROXY_HEALTH_ADDRESS": ":8081", + "NB_PROXY_LOG_LEVEL": "debug", + "NB_PROXY_PRIVATE": "true", + // Management is plain HTTP in-cluster, so allow the proxy token to + // ride a non-TLS gRPC connection. + "NB_PROXY_ALLOW_INSECURE": "true", + // The combined server multiplexes the relay over WebSocket on :8080 + // (no QUIC listener). The proxy's embedded relay client defaults to + // QUIC, which fails here and flaps the relay link, churning the + // proxy peer so it never stably registers. Force WS transport. + "NB_RELAY_TRANSPORT": "ws", + // Trace the embedded client (relay / signal / handshake) so + // peer-registration issues are visible in the proxy logs. + "NB_PROXY_CLIENT_LOG_LEVEL": "trace", + }, + HostConfigModifier: func(hc *container.HostConfig) { + hc.Binds = append(hc.Binds, workDir+":/certs") + hc.CapAdd = append(hc.CapAdd, "NET_ADMIN", "SYS_ADMIN", "SYS_RESOURCE", "NET_BIND_SERVICE") + }, + WaitingFor: wait.ForLog("Initial mapping sync complete").WithStartupTimeout(90 * time.Second), + } + + ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + return nil, fmt.Errorf("start proxy container: %w", err) + } + + return &Proxy{container: ctr, workDir: workDir}, nil +} + +// Logs returns the proxy container logs, for diagnostics on failure. +func (p *Proxy) Logs(ctx context.Context) string { + return containerLogs(ctx, p.container) +} + +// Terminate stops the proxy container and cleans its work dir. +func (p *Proxy) Terminate(ctx context.Context) error { + var err error + if p.container != nil { + err = p.container.Terminate(ctx) + } + if p.workDir != "" { + _ = os.RemoveAll(p.workDir) + } + return err +}