//go:build e2e package harness import ( "bytes" "context" "encoding/json" "fmt" "io" "os" "time" "github.com/docker/docker/api/types/container" "github.com/testcontainers/testcontainers-go" tcexec "github.com/testcontainers/testcontainers-go/exec" "github.com/testcontainers/testcontainers-go/wait" ) const ( proxyDockerfile = "proxy/Dockerfile.multistage" // defaultProxyImage is the local tag the reverse proxy is built under from // proxyDockerfile. Override with NB_E2E_PROXY_IMAGE: a value with a "/" is // pulled as a published image; a bare tag is built under that name. defaultProxyImage = "netbird-reverse-proxy:e2e" proxyAlias = "proxy" // AgentNetworkCluster is the proxy cluster the e2e provider bootstraps and // the proxy serves. It must equal the management's exposed domain // (combinedAlias) — the working manual setup uses one NETBIRD_DOMAIN for // both. The agent-network endpoint is .. AgentNetworkCluster = combinedAlias ) // Proxy is a running agent-network gateway (netbird proxy) container. type Proxy struct { container testcontainers.Container workDir string } // StartProxy builds the proxy image and runs it on the combined server's // network, registered via the given account proxy token and serving the // AgentNetworkCluster over a self-signed wildcard cert. It does not wait for // peer connectivity — callers poll management for the proxy peer. // StartProxy launches the reverse-proxy container. Optional envOverrides are // merged into the container environment after the defaults, so callers can set // or override any NB_PROXY_* var (e.g. NB_PROXY_TUNNEL_CACHE_TTL for tests that // need a short authorization-cache window). func StartProxy(ctx context.Context, c *Combined, proxyToken string, envOverrides ...map[string]string) (*Proxy, error) { root, err := repoRoot(ctx) if err != nil { return nil, err } proxyImage, err := resolveImage(ctx, root, "NB_E2E_PROXY_IMAGE", defaultProxyImage, proxyDockerfile) if err != nil { return nil, err } workDir, err := os.MkdirTemp("/tmp", "nb-e2e-proxy-*") if err != nil { return nil, fmt.Errorf("create proxy work dir: %w", err) } // MkdirTemp creates the dir 0700; widen it so the non-root proxy container // can traverse the bind-mounted cert dir on Linux CI runners. if err := os.Chmod(workDir, 0o755); err != nil { //nolint:gosec // throwaway e2e cert dir, must be traversable by the proxy container uid return nil, fmt.Errorf("chmod proxy cert dir: %w", err) } if err := writeSelfSignedCert(workDir, []string{"*." + AgentNetworkCluster, AgentNetworkCluster}); err != nil { return nil, err } req := testcontainers.ContainerRequest{ Image: proxyImage, Networks: []string{c.network.Name}, NetworkAliases: map[string][]string{c.network.Name: {proxyAlias}}, Env: map[string]string{ "NB_PROXY_TOKEN": proxyToken, "NB_PROXY_MANAGEMENT_ADDRESS": combinedExposedURL, "NB_PROXY_DOMAIN": AgentNetworkCluster, "NB_PROXY_ADDRESS": ":443", "NB_PROXY_CERTIFICATE_DIRECTORY": "/certs", "NB_PROXY_HEALTH_ADDRESS": ":8081", "NB_PROXY_LOG_LEVEL": "debug", "NB_PROXY_PRIVATE": "true", // Management is plain HTTP in-cluster, so allow the proxy token to // ride a non-TLS gRPC connection. "NB_PROXY_ALLOW_INSECURE": "true", // The combined server multiplexes the relay over WebSocket on :8080 // (no QUIC listener). The proxy's embedded relay client defaults to // QUIC, which fails here and flaps the relay link, churning the // proxy peer so it never stably registers. Force WS transport. "NB_RELAY_TRANSPORT": "ws", // Trace the embedded client (relay / signal / handshake) so // peer-registration issues are visible in the proxy logs. "NB_PROXY_CLIENT_LOG_LEVEL": "trace", }, HostConfigModifier: func(hc *container.HostConfig) { hc.Binds = append(hc.Binds, workDir+":/certs") hc.CapAdd = append(hc.CapAdd, "NET_ADMIN", "SYS_ADMIN", "SYS_RESOURCE", "NET_BIND_SERVICE") }, WaitingFor: wait.ForLog("Initial mapping sync complete").WithStartupTimeout(90 * time.Second), } for _, ov := range envOverrides { for k, v := range ov { req.Env[k] = v } } ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: req, Started: true, }) if err != nil { return nil, fmt.Errorf("start proxy container: %w", err) } return &Proxy{container: ctr, workDir: workDir}, nil } // ProxyDebugClient is one per-account embedded client the proxy runs, as the // proxy's debug endpoint reports it. type ProxyDebugClient struct { AccountID string `json:"account_id"` ServiceCount int `json:"service_count"` ServiceKeys []string `json:"service_keys"` } // DebugClients lists the per-account clients the proxy is running, through // the proxy's own debug CLI inside the container. The proxy must be started // with NB_PROXY_DEBUG_ENDPOINT=true. func (p *Proxy) DebugClients(ctx context.Context) ([]ProxyDebugClient, error) { code, reader, err := p.container.Exec(ctx, []string{"/usr/bin/netbird-proxy", "debug", "clients", "--json"}, tcexec.Multiplexed()) if err != nil { return nil, fmt.Errorf("exec debug clients: %w", err) } out, _ := io.ReadAll(reader) if code != 0 { return nil, fmt.Errorf("debug clients exited %d: %s", code, string(out)) } // stderr is multiplexed in; the JSON document starts at the first brace. start := bytes.IndexByte(out, '{') if start < 0 { return nil, fmt.Errorf("no JSON in debug clients output: %s", string(out)) } var resp struct { Clients []ProxyDebugClient `json:"clients"` } if err := json.NewDecoder(bytes.NewReader(out[start:])).Decode(&resp); err != nil { return nil, fmt.Errorf("decode debug clients output: %w", err) } return resp.Clients, nil } // Logs returns the proxy container logs, for diagnostics on failure. func (p *Proxy) Logs(ctx context.Context) string { return containerLogs(ctx, p.container) } // Terminate stops the proxy container and cleans its work dir. func (p *Proxy) Terminate(ctx context.Context) error { var err error if p.container != nil { err = p.container.Terminate(ctx) } if p.workDir != "" { _ = os.RemoveAll(p.workDir) } return err }