diff --git a/.github/workflows/agent-network-e2e.yml b/.github/workflows/agent-network-e2e.yml new file mode 100644 index 000000000..c041bfbfa --- /dev/null +++ b/.github/workflows/agent-network-e2e.yml @@ -0,0 +1,69 @@ +name: Agent Network E2E + +on: + push: + branches: + - main + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + e2e: + name: Agent Network E2E + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Install Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version-file: "go.mod" + + # Container-driver builder so the harness can build the combined/proxy/ + # client images from source with a local layer cache. + - name: Set up Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + + # Persist the Docker layer cache across runs. This caches the base, apt, + # and go-mod-download layers; the Go compile still re-runs, as BuildKit + # mount caches cannot be exported to the GitHub cache. + - name: Cache Docker layers + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-anet-e2e-buildx-${{ hashFiles('go.sum', 'combined/Dockerfile.multistage', 'proxy/Dockerfile.multistage', 'e2e/harness/Dockerfile.client') }} + restore-keys: | + ${{ runner.os }}-anet-e2e-buildx- + + - name: Run agent-network e2e + env: + # Build the images from source (this branch's code) with the shared + # local layer cache. + NB_E2E_BUILDX_CACHE: /tmp/.buildx-cache + # Provider credentials. Each provider scenario skips if its + # token (and URL, for gateways) is unset, so partial coverage is fine. + OPENAI_TOKEN: ${{ secrets.E2E_OPENAI_TOKEN }} + ANTHROPIC_TOKEN: ${{ secrets.E2E_ANTHROPIC_TOKEN }} + VERCEL_URL: ${{ secrets.E2E_VERCEL_URL }} + VERCEL_TOKEN: ${{ secrets.E2E_VERCEL_TOKEN }} + OPENROUTER_URL: ${{ secrets.E2E_OPENROUTER_URL }} + OPENROUTER_TOKEN: ${{ secrets.E2E_OPENROUTER_TOKEN }} + CLOUDFLARE_URL: ${{ secrets.E2E_CLOUDFLARE_URL }} + CLOUDFLARE_TOKEN: ${{ secrets.E2E_CLOUDFLARE_TOKEN }} + AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.E2E_AWS_BEARER_TOKEN_BEDROCK }} + AWS_REGION: ${{ secrets.E2E_AWS_REGION }} + # Vertex (Anthropic-on-Vertex): SA + project required; region defaults + # to "global", model to a pinned claude snapshot. + GOOGLE_VERTEX_SA_BASE64: ${{ secrets.E2E_GOOGLE_VERTEX_SA_BASE64 }} + GOOGLE_VERTEX_PROJECT: ${{ secrets.E2E_GOOGLE_VERTEX_PROJECT }} + GOOGLE_VERTEX_REGION: ${{ secrets.E2E_GOOGLE_VERTEX_REGION }} + GOOGLE_VERTEX_MODEL: ${{ secrets.E2E_GOOGLE_VERTEX_MODEL }} + run: go test -tags e2e -timeout 40m -v ./e2e/... diff --git a/combined/Dockerfile.multistage b/combined/Dockerfile.multistage index ef3d68c6e..79746819d 100644 --- a/combined/Dockerfile.multistage +++ b/combined/Dockerfile.multistage @@ -5,12 +5,16 @@ WORKDIR /app RUN apt-get update && apt-get install -y gcc libc6-dev git && rm -rf /var/lib/apt/lists/* COPY go.mod go.sum ./ -RUN go mod download +RUN --mount=type=cache,target=/go/pkg/mod go mod download COPY . . -# Build with version info from git (matching goreleaser ldflags) -RUN CGO_ENABLED=1 GOOS=linux go build \ +# Build with version info from git (matching goreleaser ldflags). +# BuildKit cache mounts persist the module + build caches across image builds, +# so a source change recompiles incrementally instead of from scratch. +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=1 GOOS=linux go build \ -ldflags="-s -w \ -X github.com/netbirdio/netbird/version.version=$(git describe --tags --always --dirty 2>/dev/null || echo 'dev') \ -X main.commit=$(git rev-parse --short HEAD 2>/dev/null || echo 'unknown') \ diff --git a/e2e/agentnetwork/bootstrap_test.go b/e2e/agentnetwork/bootstrap_test.go new file mode 100644 index 000000000..a73d55117 --- /dev/null +++ b/e2e/agentnetwork/bootstrap_test.go @@ -0,0 +1,30 @@ +//go:build e2e + +package agentnetwork + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestCombinedBootstrap proves Pillar 1: the shared combined server came up and +// the /api/setup-minted PAT authenticates a real management API call through +// the typed REST client (the bootstrap itself ran in TestMain). +func TestCombinedBootstrap(t *testing.T) { + ctx := context.Background() + + require.NotEmpty(t, srv.PAT, "TestMain must have minted an admin PAT") + + users, err := srv.API().Users.List(ctx) + require.NoError(t, err, "authenticated Users.List must round-trip") + require.NotEmpty(t, users, "the bootstrapped account must have at least one user") + + var emails []string + for _, u := range users { + emails = append(emails, u.Email) + } + assert.Contains(t, emails, "admin@netbird.test", "the bootstrapped owner should appear in the users list") +} diff --git a/e2e/agentnetwork/chat_test.go b/e2e/agentnetwork/chat_test.go new file mode 100644 index 000000000..5e3a79273 --- /dev/null +++ b/e2e/agentnetwork/chat_test.go @@ -0,0 +1,281 @@ +//go:build e2e + +package agentnetwork + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/e2e/harness" + "github.com/netbirdio/netbird/shared/management/http/api" +) + +// providerCase is one entry in the live provider matrix. The same scenario runs +// for every available provider; availability is keyed off env vars so the suite +// covers whatever credentials are present (source ~/.llm-keys locally / set the +// Actions secrets in CI). +type providerCase struct { + name string + catalogID string + upstream string + apiKey string + model string // body model (chat/messages) or path model@version (vertex) + kind string // harness.WireChat, harness.WireMessages, or harness.WireVertex + project string // vertex only: GCP project for the rawPredict path + region string // vertex only: GCP region for the rawPredict path +} + +// availableProviders builds the matrix from the provider env vars that are set. +func availableProviders() []providerCase { + var ps []providerCase + if k := os.Getenv("OPENAI_TOKEN"); k != "" { + ps = append(ps, providerCase{name: "openai", catalogID: "openai_api", upstream: "https://api.openai.com", apiKey: k, model: "gpt-4o-mini", kind: harness.WireChat}) + } + if k := os.Getenv("ANTHROPIC_TOKEN"); k != "" { + ps = append(ps, providerCase{name: "anthropic", catalogID: "anthropic_api", upstream: "https://api.anthropic.com", apiKey: k, model: "claude-haiku-4-5", kind: harness.WireMessages}) + } + if k, u := os.Getenv("VERCEL_TOKEN"), os.Getenv("VERCEL_URL"); k != "" && u != "" { + ps = append(ps, providerCase{name: "vercel", catalogID: "vercel_ai_gateway", upstream: u, apiKey: k, model: "openai/gpt-4o-mini", kind: harness.WireChat}) + } + if k, u := os.Getenv("OPENROUTER_TOKEN"), os.Getenv("OPENROUTER_URL"); k != "" && u != "" { + // Distinct model string from Vercel so each provider routes unambiguously + // while all are enabled together. + ps = append(ps, providerCase{name: "openrouter", catalogID: "openrouter", upstream: u, apiKey: k, model: "openai/gpt-4o", kind: harness.WireChat}) + } + if k, u := os.Getenv("CLOUDFLARE_TOKEN"), os.Getenv("CLOUDFLARE_URL"); k != "" && u != "" { + // Cloudflare AI Gateway routes by a provider segment in the URL path; + // append the openai provider unless the gateway URL already carries one. + if !strings.Contains(u, "/openai") { + u = strings.TrimRight(u, "/") + "/openai" + } + // Raw model (distinct string from OpenAI's gpt-4o-mini). + ps = append(ps, providerCase{name: "cloudflare", catalogID: "cloudflare_ai_gateway", upstream: u, apiKey: k, model: "gpt-4o", kind: harness.WireChat}) + } + // Vertex (vertex_ai_api): Anthropic-on-Vertex, path-routed, SA-OAuth + // (api_key = keyfile::). The model travels in the rawPredict path rather + // than the body, so the provider is created without a models array. Region + // defaults to "global" (host aiplatform.googleapis.com); a real region uses + // -aiplatform.googleapis.com. + if sa := os.Getenv("GOOGLE_VERTEX_SA_BASE64"); sa != "" { + project := os.Getenv("GOOGLE_VERTEX_PROJECT") + if project != "" { + region := os.Getenv("GOOGLE_VERTEX_REGION") + if region == "" { + region = "global" + } + host := "aiplatform.googleapis.com" + if region != "global" { + host = region + "-aiplatform.googleapis.com" + } + model := os.Getenv("GOOGLE_VERTEX_MODEL") + if model == "" { + model = "claude-sonnet-4-5@20250929" + } + ps = append(ps, providerCase{ + name: "vertex", catalogID: "vertex_ai_api", upstream: "https://" + host, + apiKey: "keyfile::" + sa, model: model, kind: harness.WireVertex, + project: project, region: region, + }) + } + } + + // Bedrock: path-routed, bearer auth. Model is a cross-region inference + // profile id (distinct string from the first-party Anthropic case). + if k := os.Getenv("AWS_BEARER_TOKEN_BEDROCK"); k != "" { + region := os.Getenv("AWS_REGION") + if region == "" { + region = "us-east-1" + } + ps = append(ps, providerCase{name: "bedrock", catalogID: "bedrock_api", upstream: "https://bedrock-runtime." + region + ".amazonaws.com", apiKey: k, model: "us.anthropic.claude-haiku-4-5", kind: harness.WireMessages}) + } + return ps +} + +// providerRequest builds a create request for a matrix provider: enabled, with +// a uniquely-priced model for body-routed providers and none for the +// path-routed Vertex (whose model lives in the request path). +func providerRequest(pc providerCase) api.AgentNetworkProviderRequest { + req := api.AgentNetworkProviderRequest{ + Name: pc.name, + ProviderId: pc.catalogID, + UpstreamUrl: pc.upstream, + ApiKey: &pc.apiKey, + Enabled: ptr(true), + } + if pc.kind != harness.WireVertex { + req.Models = &[]api.AgentNetworkProviderModel{ + {Id: pc.model, InputPer1k: 0.001, OutputPer1k: 0.002}, + } + } + return req +} + +// TestProvidersMatrix is Pillar 3: it provisions every available provider (all +// enabled, each with a unique model so routing stays unambiguous), runs proxy + +// client once, and drives the same live chat-completion scenario through each +// provider over the WireGuard tunnel. Each provider must return 200 and produce +// an ingested access-log row. +func TestProvidersMatrix(t *testing.T) { + matrix := availableProviders() + if len(matrix) == 0 { + t.Skip("no provider keys set; source ~/.llm-keys to run the provider matrix") + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) + defer cancel() + + // Group + setup key the client joins into; the policy authorizes it. + grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-agents"}) + require.NoError(t, err, "create agents group") + t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) }) + + ephemeral := false + sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{ + Name: "e2e-client", + Type: "reusable", + ExpiresIn: 86400, + UsageLimit: 0, + AutoGroups: []string{grp.Id}, + Ephemeral: &ephemeral, + }) + require.NoError(t, err, "mint setup key") + require.NotEmpty(t, sk.Key, "setup key plaintext") + + // Create every provider, all enabled, each with a unique model string so the + // proxy's connect-time snapshot carries them all and model→provider routing + // is unambiguous (provider toggles after connect don't reconcile to the + // proxy, so we enable everything up front). The first create bootstraps the + // cluster. + ids := make([]string, 0, len(matrix)) + for i, pc := range matrix { + req := providerRequest(pc) + if i == 0 { + req.BootstrapCluster = ptr(harness.AgentNetworkCluster) + } + prov, perr := srv.CreateProvider(ctx, req) + require.NoError(t, perr, "create provider %s", pc.name) + ids = append(ids, prov.Id) + id := prov.Id + t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), id) }) + } + + enabled := true + pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{ + Name: "e2e-allow", + Enabled: &enabled, + SourceGroups: []string{grp.Id}, + DestinationProviderIds: ids, + // Token limit at the 60s window floor with caps far above the few hundred + // tokens this suite drives, so it never blocks traffic but switches on + // usage metering, which is what makes consumption rows get recorded. + Limits: &api.AgentNetworkPolicyLimits{ + TokenLimit: api.AgentNetworkPolicyTokenLimit{ + Enabled: true, + GroupCap: 10_000_000, + UserCap: 10_000_000, + WindowSeconds: 60, + }, + }, + }) + require.NoError(t, err, "create policy") + t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) }) + + settings, err := srv.GetSettings(ctx) + require.NoError(t, err, "read settings for endpoint") + require.NotEmpty(t, settings.Endpoint, "agent-network endpoint must be assigned") + + // Proxy (global CLI token) + client, brought up once. + proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-proxy") + require.NoError(t, err, "mint proxy token via CLI") + px, err := harness.StartProxy(ctx, srv, proxyToken) + require.NoError(t, err, "start proxy") + t.Cleanup(func() { _ = px.Terminate(context.Background()) }) + + cl, err := harness.StartClient(ctx, srv, sk.Key) + require.NoError(t, err, "start client") + t.Cleanup(func() { _ = cl.Terminate(context.Background()) }) + + require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management") + if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil { + t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background())) + } + proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint) + require.NoError(t, err, "resolve agent-network endpoint to proxy IP") + + for _, pc := range matrix { + pc := pc + t.Run(pc.name, func(t *testing.T) { + before, _ := srv.ListAccessLogs(ctx) + + // Unique per provider so we can find this provider's row by its + // session id and confirm the marker propagated end-to-end. + sessionID := "e2e-session-" + pc.name + + // Retry briefly to absorb tunnel/DNS jitter on the first call. + var code int + var body string + deadline := time.Now().Add(90 * time.Second) + for time.Now().Before(deadline) { + var c int + var b string + var cerr error + if pc.kind == harness.WireVertex { + c, b, cerr = cl.Vertex(ctx, settings.Endpoint, proxyIP, pc.project, pc.region, pc.model, "Reply with exactly: pong", sessionID) + } else { + c, b, cerr = cl.Chat(ctx, settings.Endpoint, proxyIP, pc.kind, pc.model, "Reply with exactly: pong", sessionID) + } + if cerr == nil { + code, body = c, b + if code == 200 { + break + } + } + time.Sleep(5 * time.Second) + } + require.Equal(t, 200, code, "chat through %s (%s %s) should return 200; body: %s", pc.name, pc.kind, pc.model, body) + + require.Eventually(t, func() bool { + logs, lerr := srv.ListAccessLogs(ctx) + return lerr == nil && logs.TotalRecords > before.TotalRecords + }, 30*time.Second, 2*time.Second, "an access-log row should be ingested for %s", pc.name) + + // The session id sent as x-session-id must round-trip into the + // access-log row for this provider. + require.Eventually(t, func() bool { + logs, lerr := srv.ListAccessLogs(ctx) + if lerr != nil { + return false + } + for _, r := range logs.Data { + if r.SessionId != nil && *r.SessionId == sessionID { + return true + } + } + return false + }, 30*time.Second, 2*time.Second, "session id %q must be recorded in an access-log row for %s", sessionID, pc.name) + }) + } + + // Metering: the policy's uncapped token limit switches on usage recording, + // so the live traffic just driven must surface as consumption rows with + // positive token counts. Consumption is account-scoped (keyed by source + // group / user and time window, not per provider), and ingest is async, so + // poll for any row that has booked tokens. + require.Eventually(t, func() bool { + rows, lerr := srv.ListConsumption(ctx) + if lerr != nil { + return false + } + for _, r := range rows { + if r.TokensInput > 0 && r.TokensOutput > 0 { + return true + } + } + return false + }, 60*time.Second, 3*time.Second, "consumption must be recorded with positive token counts after live traffic") +} diff --git a/e2e/agentnetwork/main_test.go b/e2e/agentnetwork/main_test.go new file mode 100644 index 000000000..17c5e00be --- /dev/null +++ b/e2e/agentnetwork/main_test.go @@ -0,0 +1,46 @@ +//go:build e2e + +// Package agentnetwork holds the container-based agent-network e2e suite. A +// single combined server is built and bootstrapped once per package run +// (TestMain) and shared across tests via srv; each test creates and cleans up +// its own resources so order doesn't matter. +package agentnetwork + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/netbirdio/netbird/e2e/harness" +) + +// srv is the shared combined server for the package, ready (PAT-authenticated) +// by the time any Test runs. +var srv *harness.Combined + +func TestMain(m *testing.M) { + os.Exit(run(m)) +} + +func run(m *testing.M) int { + // Generous timeout to cover a cold image build on first run. + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer cancel() + + var err error + srv, err = harness.StartCombined(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "e2e: start combined server: %v\n", err) + return 1 + } + defer func() { _ = srv.Terminate(context.Background()) }() + + if _, err := srv.Bootstrap(ctx); err != nil { + fmt.Fprintf(os.Stderr, "e2e: bootstrap admin PAT: %v\n", err) + return 1 + } + + return m.Run() +} diff --git a/e2e/agentnetwork/management_test.go b/e2e/agentnetwork/management_test.go new file mode 100644 index 000000000..cfd03f63c --- /dev/null +++ b/e2e/agentnetwork/management_test.go @@ -0,0 +1,221 @@ +//go:build e2e + +package agentnetwork + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/netbirdio/netbird/e2e/harness" + "github.com/netbirdio/netbird/shared/management/client/rest" + "github.com/netbirdio/netbird/shared/management/http/api" +) + +func ptr[T any](v T) *T { return &v } + +// newProvider creates an OpenAI-catalog provider with a dummy key (these tests +// never call the upstream) and registers cleanup. +func newProvider(t *testing.T, ctx context.Context, name string) api.AgentNetworkProvider { + t.Helper() + prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ + Name: name, + ProviderId: "openai_api", + UpstreamUrl: "https://api.openai.com", + ApiKey: ptr("sk-dummy-e2e-key"), + BootstrapCluster: ptr("eu.proxy.netbird.test"), + }) + require.NoError(t, err, "create provider %q", name) + t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) + return prov +} + +// requireClientError asserts err is a REST APIError with a 4xx status. +func requireClientError(t *testing.T, err error) { + t.Helper() + var apiErr *rest.APIError + require.ErrorAs(t, err, &apiErr, "expected a REST APIError") + assert.GreaterOrEqual(t, apiErr.StatusCode, 400, "expected a 4xx status") + assert.Less(t, apiErr.StatusCode, 500, "expected a 4xx status") +} + +// TestProviderLifecycle covers create → get → list → delete → 404 for every +// available real provider catalog (and a synthetic OpenAI provider when no +// provider keys are set), so each catalog's create and field round-trip is +// exercised. Create is offline — no upstream call — so this stays fast and +// burns no provider quota. +func TestProviderLifecycle(t *testing.T) { + ctx := context.Background() + + cases := availableProviders() + if len(cases) == 0 { + cases = []providerCase{{ + name: "openai", catalogID: "openai_api", upstream: "https://api.openai.com", + apiKey: "sk-dummy-e2e-key", model: "gpt-4o-mini", kind: harness.WireChat, + }} + } + + for i, pc := range cases { + i, pc := i, pc + t.Run(pc.name, func(t *testing.T) { + req := providerRequest(pc) + req.Name = "lc-" + pc.name + // Bootstrap the cluster on the first create in case the matrix has + // not run (e.g. no provider keys → settings not yet bootstrapped). + if i == 0 { + req.BootstrapCluster = ptr(harness.AgentNetworkCluster) + } + + prov, err := srv.CreateProvider(ctx, req) + require.NoError(t, err, "create %s provider", pc.name) + t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) + + assert.NotEmpty(t, prov.Id, "created provider must have an id") + assert.Equal(t, pc.catalogID, prov.ProviderId, "catalog id must round-trip") + assert.Equal(t, req.Name, prov.Name, "name must round-trip") + assert.Equal(t, pc.upstream, prov.UpstreamUrl, "upstream must round-trip") + + got, err := srv.GetProvider(ctx, prov.Id) + require.NoError(t, err, "get provider") + assert.Equal(t, prov.Id, got.Id) + + list, err := srv.ListProviders(ctx) + require.NoError(t, err, "list providers") + var ids []string + for _, p := range list { + ids = append(ids, p.Id) + } + assert.Contains(t, ids, prov.Id, "created provider must appear in the list") + + require.NoError(t, srv.DeleteProvider(ctx, prov.Id), "delete provider") + _, err = srv.GetProvider(ctx, prov.Id) + requireClientError(t, err) + }) + } +} + +// TestProviderValidation exercises the create-time validation rules. These are +// uniform across catalogs (no per-provider required-field rules exist: a +// catalog-specific malformed value such as a Vertex key without the keyfile:: +// prefix is accepted at create and only fails at the proxy), so the cases here +// are catalog-agnostic: missing API key, unknown catalog id, an invalid upstream +// URL, and a blank name. +func TestProviderValidation(t *testing.T) { + ctx := context.Background() + + _, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ + Name: "No Key", + ProviderId: "openai_api", + UpstreamUrl: "https://api.openai.com", + }) + requireClientError(t, err) + + _, err = srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ + Name: "Unknown Catalog", + ProviderId: "totally_unknown_provider", + UpstreamUrl: "https://example.com", + ApiKey: ptr("sk-dummy"), + }) + requireClientError(t, err) + + _, err = srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ + Name: "Bad Upstream", + ProviderId: "openai_api", + UpstreamUrl: "not-a-url", + ApiKey: ptr("sk-dummy"), + }) + requireClientError(t, err) + + _, err = srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ + Name: " ", + ProviderId: "openai_api", + UpstreamUrl: "https://api.openai.com", + ApiKey: ptr("sk-dummy"), + }) + requireClientError(t, err) +} + +// TestSettingsRoundTrip flips the collection toggles and confirms cluster / +// subdomain stay immutable, then restores the original state. +func TestSettingsRoundTrip(t *testing.T) { + ctx := context.Background() + + // Settings are bootstrapped on first provider create. + newProvider(t, ctx, "Settings Bootstrap") + + before, err := srv.GetSettings(ctx) + require.NoError(t, err, "get settings") + require.NotEmpty(t, before.Cluster, "settings must carry an assigned cluster") + + flipped, err := srv.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ + EnableLogCollection: !before.EnableLogCollection, + EnablePromptCollection: !before.EnablePromptCollection, + RedactPii: !before.RedactPii, + }) + require.NoError(t, err, "update settings") + assert.Equal(t, !before.EnableLogCollection, flipped.EnableLogCollection, "log collection toggle must flip") + assert.Equal(t, !before.EnablePromptCollection, flipped.EnablePromptCollection, "prompt collection toggle must flip") + assert.Equal(t, before.Cluster, flipped.Cluster, "cluster must be immutable across updates") + assert.Equal(t, before.Subdomain, flipped.Subdomain, "subdomain must be immutable across updates") + + // Restore the original toggles. + _, err = srv.UpdateSettings(ctx, api.AgentNetworkSettingsRequest{ + EnableLogCollection: before.EnableLogCollection, + EnablePromptCollection: before.EnablePromptCollection, + RedactPii: before.RedactPii, + }) + require.NoError(t, err, "restore settings") +} + +// TestPolicyWindowFloor rejects an enabled limit below the 60s window floor and +// accepts one at the floor. +func TestPolicyWindowFloor(t *testing.T) { + ctx := context.Background() + + grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-policy-grp"}) + require.NoError(t, err, "create source group") + t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) }) + + prov := newProvider(t, ctx, "Policy Provider") + + limits := func(window int64) *api.AgentNetworkPolicyLimits { + return &api.AgentNetworkPolicyLimits{ + TokenLimit: api.AgentNetworkPolicyTokenLimit{ + Enabled: true, + GroupCap: 1000, + UserCap: 1000, + WindowSeconds: window, + }, + } + } + + _, err = srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{ + Name: "e2e-below-floor", + SourceGroups: []string{grp.Id}, + DestinationProviderIds: []string{prov.Id}, + Limits: limits(30), + }) + requireClientError(t, err) + + pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{ + Name: "e2e-at-floor", + SourceGroups: []string{grp.Id}, + DestinationProviderIds: []string{prov.Id}, + Limits: limits(60), + }) + require.NoError(t, err, "policy at the 60s floor must be accepted") + assert.NotEmpty(t, pol.Id, "created policy must have an id") + t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) }) +} + +// TestConsumptionList confirms the read endpoint always returns an array, never +// a 404/500. +func TestConsumptionList(t *testing.T) { + ctx := context.Background() + + rows, err := srv.ListConsumption(ctx) + require.NoError(t, err, "consumption list must not error") + assert.NotNil(t, rows, "consumption must be a JSON array (possibly empty)") +} diff --git a/e2e/harness/Dockerfile.client b/e2e/harness/Dockerfile.client new file mode 100644 index 000000000..114577d60 --- /dev/null +++ b/e2e/harness/Dockerfile.client @@ -0,0 +1,24 @@ +# Multistage build for the NetBird client used in e2e tests. The repo has no +# source-building client Dockerfile (client/Dockerfile packages a goreleaser +# artifact), so this mirrors its alpine runtime + entrypoint while compiling the +# CGO-free client inline. BuildKit cache mounts keep rebuilds incremental. + +FROM golang:1.25-bookworm AS builder +WORKDIR /src +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download +COPY . . +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=linux go build -o /out/netbird ./client + +FROM alpine:3.24 +RUN apk add --no-cache bash ca-certificates ip6tables iproute2 iptables +ENV NETBIRD_BIN="/usr/local/bin/netbird" \ + NB_LOG_FILE="console,/var/log/netbird/client.log" \ + NB_DAEMON_ADDR="unix:///var/run/netbird.sock" \ + NB_ENABLE_CAPTURE="false" \ + NB_ENTRYPOINT_SERVICE_TIMEOUT="30" +ENTRYPOINT [ "/usr/local/bin/netbird-entrypoint.sh" ] +COPY client/netbird-entrypoint.sh /usr/local/bin/netbird-entrypoint.sh +COPY --from=builder /out/netbird /usr/local/bin/netbird diff --git a/e2e/harness/agentnetwork.go b/e2e/harness/agentnetwork.go new file mode 100644 index 000000000..192385ab1 --- /dev/null +++ b/e2e/harness/agentnetwork.go @@ -0,0 +1,130 @@ +//go:build e2e + +package harness + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/netbirdio/netbird/shared/management/http/api" +) + +// The shared REST client doesn't (yet) expose typed agent-network methods, so +// these helpers drive the /api/agent-network/* endpoints through the client's +// NewRequest primitive — reusing its auth, error handling (rest.APIError on +// non-2xx), and transport — while still speaking the generated api types. + +// anRequest issues an agent-network API call and decodes the JSON response into +// T. A non-2xx response surfaces as a *rest.APIError from the client, which +// tests inspect for negative-path status assertions. +func anRequest[T any](ctx context.Context, c *Combined, method, path string, body any) (T, error) { + var out T + var reader io.Reader + if body != nil { + bs, err := json.Marshal(body) + if err != nil { + return out, fmt.Errorf("marshal %s %s: %w", method, path, err) + } + reader = bytes.NewReader(bs) + } + + resp, err := c.api.NewRequest(ctx, method, path, reader, nil) + if err != nil { + return out, err + } + defer resp.Body.Close() + + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return out, fmt.Errorf("decode %s %s response: %w", method, path, err) + } + return out, nil +} + +// anDelete issues a DELETE and discards the (empty-object) body. +func anDelete(ctx context.Context, c *Combined, path string) error { + resp, err := c.api.NewRequest(ctx, http.MethodDelete, path, nil, nil) + if err != nil { + return err + } + resp.Body.Close() + return nil +} + +// CreateProvider creates an agent-network provider. +func (c *Combined) CreateProvider(ctx context.Context, req api.AgentNetworkProviderRequest) (api.AgentNetworkProvider, error) { + return anRequest[api.AgentNetworkProvider](ctx, c, http.MethodPost, "/api/agent-network/providers", req) +} + +// GetProvider fetches a provider by id. +func (c *Combined) GetProvider(ctx context.Context, id string) (api.AgentNetworkProvider, error) { + return anRequest[api.AgentNetworkProvider](ctx, c, http.MethodGet, "/api/agent-network/providers/"+id, nil) +} + +// ListProviders returns all providers for the account. +func (c *Combined) ListProviders(ctx context.Context) ([]api.AgentNetworkProvider, error) { + return anRequest[[]api.AgentNetworkProvider](ctx, c, http.MethodGet, "/api/agent-network/providers", nil) +} + +// DeleteProvider removes a provider by id. +func (c *Combined) DeleteProvider(ctx context.Context, id string) error { + return anDelete(ctx, c, "/api/agent-network/providers/"+id) +} + +// SetProviderEnabled toggles a provider's enabled flag, preserving its other +// fields (the API key is omitted, which keeps the stored one). Used to run one +// provider at a time so model→provider routing is unambiguous. +func (c *Combined) SetProviderEnabled(ctx context.Context, id string, enabled bool) error { + p, err := c.GetProvider(ctx, id) + if err != nil { + return err + } + _, err = anRequest[api.AgentNetworkProvider](ctx, c, http.MethodPut, "/api/agent-network/providers/"+id, api.AgentNetworkProviderRequest{ + Name: p.Name, + ProviderId: p.ProviderId, + UpstreamUrl: p.UpstreamUrl, + Enabled: &enabled, + Models: &p.Models, + }) + return err +} + +// CreatePolicy creates an agent-network policy. +func (c *Combined) CreatePolicy(ctx context.Context, req api.AgentNetworkPolicyRequest) (api.AgentNetworkPolicy, error) { + return anRequest[api.AgentNetworkPolicy](ctx, c, http.MethodPost, "/api/agent-network/policies", req) +} + +// UpdatePolicy replaces a policy by id. +func (c *Combined) UpdatePolicy(ctx context.Context, id string, req api.AgentNetworkPolicyRequest) (api.AgentNetworkPolicy, error) { + return anRequest[api.AgentNetworkPolicy](ctx, c, http.MethodPut, "/api/agent-network/policies/"+id, req) +} + +// DeletePolicy removes a policy by id. +func (c *Combined) DeletePolicy(ctx context.Context, id string) error { + return anDelete(ctx, c, "/api/agent-network/policies/"+id) +} + +// GetSettings returns the account's agent-network settings row. It exists only +// after the first provider create bootstraps it. +func (c *Combined) GetSettings(ctx context.Context) (api.AgentNetworkSettings, error) { + return anRequest[api.AgentNetworkSettings](ctx, c, http.MethodGet, "/api/agent-network/settings", nil) +} + +// UpdateSettings applies the mutable collection toggles. +func (c *Combined) UpdateSettings(ctx context.Context, req api.AgentNetworkSettingsRequest) (api.AgentNetworkSettings, error) { + return anRequest[api.AgentNetworkSettings](ctx, c, http.MethodPut, "/api/agent-network/settings", req) +} + +// ListConsumption returns the account's consumption rows (possibly empty). +func (c *Combined) ListConsumption(ctx context.Context) ([]api.AgentNetworkConsumption, error) { + return anRequest[[]api.AgentNetworkConsumption](ctx, c, http.MethodGet, "/api/agent-network/consumption", nil) +} + +// ListAccessLogs returns the account's agent-network access-log page (the +// flattened per-request rows the proxy ships and management ingests). +func (c *Combined) ListAccessLogs(ctx context.Context) (api.AgentNetworkAccessLogsResponse, error) { + return anRequest[api.AgentNetworkAccessLogsResponse](ctx, c, http.MethodGet, "/api/agent-network/access-logs", nil) +} diff --git a/e2e/harness/bootstrap.go b/e2e/harness/bootstrap.go new file mode 100644 index 000000000..defa03c14 --- /dev/null +++ b/e2e/harness/bootstrap.go @@ -0,0 +1,47 @@ +//go:build e2e + +package harness + +import ( + "context" + "fmt" + + "github.com/netbirdio/netbird/shared/management/client/rest" + "github.com/netbirdio/netbird/shared/management/http/api" +) + +// Bootstrap creates the initial admin owner through the unauthenticated +// /api/setup endpoint and returns the plaintext admin PAT. It also wires an +// authenticated REST client on the Combined (see API). create_pat requires the +// server to run with NB_SETUP_PAT_ENABLED=true, which the harness sets. A +// second call returns an error (the server reports setup already completed). +func (c *Combined) Bootstrap(ctx context.Context) (string, error) { + // The setup endpoint is unauthenticated; use a tokenless client. + setupClient := rest.NewWithOptions(rest.WithManagementURL(c.BaseURL)) + + createPAT := true + expireDays := 1 + resp, err := setupClient.Instance.Setup(ctx, api.PostApiSetupJSONRequestBody{ //nolint:gosec // static throwaway test credentials + Email: "admin@netbird.test", + Password: "Netbird-e2e-Passw0rd!", + Name: "E2E Admin", + CreatePat: &createPAT, + PatExpireIn: &expireDays, + }) + if err != nil { + return "", fmt.Errorf("instance setup: %w", err) + } + if resp.PersonalAccessToken == nil || *resp.PersonalAccessToken == "" { + return "", fmt.Errorf("setup succeeded but no PAT returned (is NB_SETUP_PAT_ENABLED set?)") + } + + c.PAT = *resp.PersonalAccessToken + c.api = rest.New(c.BaseURL, c.PAT) + return c.PAT, nil +} + +// API returns the PAT-authenticated management REST client. It is nil until +// Bootstrap runs. +func (c *Combined) API() *rest.Client { + return c.api +} diff --git a/e2e/harness/cert.go b/e2e/harness/cert.go new file mode 100644 index 000000000..c8a28e470 --- /dev/null +++ b/e2e/harness/cert.go @@ -0,0 +1,66 @@ +//go:build e2e + +package harness + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "os" + "path/filepath" + "time" +) + +// writeSelfSignedCert generates a self-signed TLS cert/key pair covering the +// given DNS names and writes them as tls.crt / tls.key in dir. The proxy serves +// this for the agent-network endpoint; the client curls with -k, so validity +// chains don't matter — the proxy just needs a usable cert to present. +func writeSelfSignedCert(dir string, dnsNames []string) error { + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return fmt.Errorf("generate key: %w", err) + } + + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return fmt.Errorf("generate serial: %w", err) + } + + tmpl := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: dnsNames[0]}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: dnsNames, + BasicConstraintsValid: true, + } + + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv) + if err != nil { + return fmt.Errorf("create certificate: %w", err) + } + + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + if err := os.WriteFile(filepath.Join(dir, "tls.crt"), certPEM, 0o644); err != nil { //nolint:gosec // public cert, bind-mounted and read by the proxy container + return fmt.Errorf("write cert: %w", err) + } + + keyDER, err := x509.MarshalECPrivateKey(priv) + if err != nil { + return fmt.Errorf("marshal key: %w", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + // World-readable so the (non-root) proxy container can read the bind-mounted + // key on Linux CI runners; this is a throwaway self-signed e2e key. + if err := os.WriteFile(filepath.Join(dir, "tls.key"), keyPEM, 0o644); err != nil { //nolint:gosec // throwaway self-signed e2e key, must be readable by the proxy container uid + return fmt.Errorf("write key: %w", err) + } + return nil +} diff --git a/e2e/harness/client.go b/e2e/harness/client.go new file mode 100644 index 000000000..cf7ef8945 --- /dev/null +++ b/e2e/harness/client.go @@ -0,0 +1,256 @@ +//go:build e2e + +package harness + +import ( + "context" + "fmt" + "io" + "os/exec" + "strings" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/testcontainers/testcontainers-go" + tcexec "github.com/testcontainers/testcontainers-go/exec" +) + +const ( + clientDockerfile = "e2e/harness/Dockerfile.client" + // defaultClientImage is the local tag the client is built under from + // clientDockerfile. Override with NB_E2E_CLIENT_IMAGE: a value with a "/" is + // pulled as a published image; a bare tag is built under that name. + defaultClientImage = "netbird-client:e2e" + clientAlias = "client" + curlImage = "curlimages/curl:latest" +) + +// Client is a running NetBird client container joined to the combined server. +type Client struct { + container testcontainers.Container +} + +// StartClient builds the client image and runs it on the combined server's +// network, joining via the given setup key. The image entrypoint brings the +// daemon up automatically; callers wait for connectivity with WaitConnected / +// WaitProxyPeer. +func StartClient(ctx context.Context, c *Combined, setupKey string) (*Client, error) { + root, err := repoRoot() + if err != nil { + return nil, err + } + clientImage, err := resolveImage(ctx, root, "NB_E2E_CLIENT_IMAGE", defaultClientImage, clientDockerfile) + if err != nil { + return nil, err + } + + req := testcontainers.ContainerRequest{ + Image: clientImage, + Networks: []string{c.network.Name}, + NetworkAliases: map[string][]string{c.network.Name: {clientAlias}}, + Env: map[string]string{ + "NB_MANAGEMENT_URL": combinedExposedURL, + "NB_SETUP_KEY": setupKey, + "NB_LOG_LEVEL": "info", + // Match the proxy: the combined relay is WebSocket-only, so the + // client must use WS transport to keep a stable relay link to it. + "NB_RELAY_TRANSPORT": "ws", + }, + HostConfigModifier: func(hc *container.HostConfig) { + hc.CapAdd = append(hc.CapAdd, "NET_ADMIN", "SYS_ADMIN", "SYS_RESOURCE") + }, + } + + ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + return nil, fmt.Errorf("start client container: %w", err) + } + return &Client{container: ctr}, nil +} + +// Restart bounces the client connection (netbird down/up) so it pulls a fresh +// network map — the documented workaround for a freshly-joined client not yet +// seeing a synthesized agent-network service. +func (cl *Client) Restart(ctx context.Context) error { + if _, _, err := cl.container.Exec(ctx, []string{"netbird", "down"}, tcexec.Multiplexed()); err != nil { + return fmt.Errorf("netbird down: %w", err) + } + time.Sleep(2 * time.Second) + code, reader, err := cl.container.Exec(ctx, []string{"netbird", "up"}, tcexec.Multiplexed()) + if err != nil { + return fmt.Errorf("netbird up: %w", err) + } + if code != 0 { + out, _ := io.ReadAll(reader) + return fmt.Errorf("netbird up exited %d: %s", code, string(out)) + } + return nil +} + +// Status returns `netbird status` output from inside the client. +func (cl *Client) Status(ctx context.Context) (string, error) { + code, reader, err := cl.container.Exec(ctx, []string{"netbird", "status"}, tcexec.Multiplexed()) + if err != nil { + return "", err + } + out, _ := io.ReadAll(reader) + if code != 0 { + return string(out), fmt.Errorf("netbird status exited %d", code) + } + return string(out), nil +} + +// WaitConnected polls until the client reports Management: Connected. +func (cl *Client) WaitConnected(ctx context.Context, timeout time.Duration) error { + return cl.pollStatus(ctx, timeout, "Management: Connected") +} + +// WaitProxyPeer polls until the client sees the proxy peer connected (1/1). +func (cl *Client) WaitProxyPeer(ctx context.Context, timeout time.Duration) error { + return cl.pollStatus(ctx, timeout, "1/1 Connected") +} + +func (cl *Client) pollStatus(ctx context.Context, timeout time.Duration, want string) error { + deadline := time.Now().Add(timeout) + var last string + for time.Now().Before(deadline) { + out, _ := cl.Status(ctx) + last = out + if strings.Contains(out, want) { + return nil + } + time.Sleep(3 * time.Second) + } + return fmt.Errorf("timed out waiting for %q; last status:\n%s", want, last) +} + +// ResolveProxyIP resolves the agent-network endpoint to the proxy peer's +// NetBird IP from inside the client (via magic DNS). +func (cl *Client) ResolveProxyIP(ctx context.Context, endpoint string) (string, error) { + code, reader, err := cl.container.Exec(ctx, []string{"getent", "hosts", endpoint}, tcexec.Multiplexed()) + if err != nil { + return "", err + } + out, _ := io.ReadAll(reader) + if code != 0 { + return "", fmt.Errorf("getent hosts %s exited %d", endpoint, code) + } + fields := strings.Fields(string(out)) + if len(fields) == 0 { + return "", fmt.Errorf("no address for %s", endpoint) + } + return fields[0], nil +} + +// Wire shapes for Chat. +const ( + // WireChat is the OpenAI-compatible /v1/chat/completions shape. + WireChat = "chat" + // WireMessages is the Anthropic /v1/messages shape. + WireMessages = "messages" + // WireVertex is the Anthropic-on-Vertex rawPredict shape: the client posts + // the full Vertex model path and the proxy mints the SA OAuth token. + WireVertex = "vertex" +) + +// Chat issues a chat-completion POST to the agent-network endpoint over the +// client's tunnel, returning the HTTP status and response body. kind selects +// the wire shape: WireChat (OpenAI) or WireMessages (Anthropic). A non-empty +// sessionID is sent as the universal x-session-id header the proxy records. +func (cl *Client) Chat(ctx context.Context, endpoint, proxyIP, kind, model, prompt, sessionID string) (int, string, error) { + var path, body string + var headers []string + switch kind { + case WireMessages: + path = "/v1/messages" + headers = []string{"anthropic-version: 2023-06-01"} + body = fmt.Sprintf(`{"model":%q,"max_tokens":64,"messages":[{"role":"user","content":%q}]}`, model, prompt) + default: + path = "/v1/chat/completions" + body = fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":%q}]}`, model, prompt) + } + return cl.post(ctx, endpoint, proxyIP, path, body, withSessionID(headers, sessionID)) +} + +// Vertex issues an Anthropic-on-Vertex rawPredict POST over the tunnel. Unlike +// Chat, the model is carried in the request path (project/region/model), so the +// proxy routes by path and mints the service-account OAuth token; the body uses +// the Vertex anthropic_version rather than a model field. A non-empty sessionID +// is sent as the universal x-session-id header the proxy records. +func (cl *Client) Vertex(ctx context.Context, endpoint, proxyIP, project, region, model, prompt, sessionID string) (int, string, error) { + path := fmt.Sprintf("/v1/projects/%s/locations/%s/publishers/anthropic/models/%s:rawPredict", project, region, model) + body := fmt.Sprintf(`{"anthropic_version":"vertex-2023-10-16","max_tokens":64,"messages":[{"role":"user","content":%q}]}`, prompt) + return cl.post(ctx, endpoint, proxyIP, path, body, withSessionID(nil, sessionID)) +} + +// withSessionID appends the x-session-id header when sessionID is non-empty. +func withSessionID(headers []string, sessionID string) []string { + if sessionID == "" { + return headers + } + return append(headers, "x-session-id: "+sessionID) +} + +// post runs curl in a throwaway container sharing the client's network +// namespace so the request traverses the WireGuard tunnel, pinning the endpoint +// to the proxy IP. It returns the HTTP status and response body. +func (cl *Client) post(ctx context.Context, endpoint, proxyIP, path, body string, extraHeaders []string) (int, string, error) { + url := "https://" + endpoint + path + args := []string{ + "run", "--rm", + "--network", "container:" + cl.container.GetContainerID(), + curlImage, + "-sk", "--connect-timeout", "5", "--max-time", "90", + "--resolve", endpoint + ":443:" + proxyIP, + "-o", "/dev/stderr", "-w", "%{http_code}", + "-X", "POST", url, + "-H", "Content-Type: application/json", + } + for _, h := range extraHeaders { + args = append(args, "-H", h) + } + args = append(args, "--data", body) + cmd := exec.CommandContext(ctx, "docker", args...) + // -w writes the status code to stdout; -o /dev/stderr writes the body to + // stderr so we can capture both separately. + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return 0, stderr.String(), fmt.Errorf("curl through tunnel: %w", err) + } + + code := 0 + _, _ = fmt.Sscanf(strings.TrimSpace(stdout.String()), "%d", &code) + return code, stderr.String(), nil +} + +// Logs returns the client container logs, for diagnostics on failure. +func (cl *Client) Logs(ctx context.Context) string { + return containerLogs(ctx, cl.container) +} + +// Terminate stops the client container. +func (cl *Client) Terminate(ctx context.Context) error { + if cl.container == nil { + return nil + } + return cl.container.Terminate(ctx) +} + +// containerLogs reads up to 256 KiB of a container's logs for diagnostics. +func containerLogs(ctx context.Context, c testcontainers.Container) string { + if c == nil { + return "" + } + r, err := c.Logs(ctx) + if err != nil { + return fmt.Sprintf("", err) + } + defer r.Close() + b, _ := io.ReadAll(io.LimitReader(r, 256<<10)) + return string(b) +} diff --git a/e2e/harness/combined.go b/e2e/harness/combined.go new file mode 100644 index 000000000..a6f43a139 --- /dev/null +++ b/e2e/harness/combined.go @@ -0,0 +1,243 @@ +//go:build e2e + +package harness + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/docker/go-connections/nat" + "github.com/testcontainers/testcontainers-go" + tcexec "github.com/testcontainers/testcontainers-go/exec" + "github.com/testcontainers/testcontainers-go/network" + "github.com/testcontainers/testcontainers-go/wait" + + "github.com/netbirdio/netbird/shared/management/client/rest" +) + +const ( + combinedDockerfile = "combined/Dockerfile.multistage" + // defaultCombinedImage is the local tag the combined server is built under + // from combinedDockerfile, so the e2e exercises this branch's code. Override + // with NB_E2E_COMBINED_IMAGE: a value containing a "/" is pulled as a + // published image; a bare tag is built under that name instead. + defaultCombinedImage = "netbird-combined:e2e" + combinedHTTPPort = "8080/tcp" + + // combinedAlias is the combined server's network alias AND the deployment + // domain. The working manual setup uses a single NETBIRD_DOMAIN for the + // management exposed address, the proxy domain, and the agent-network + // cluster — so we mirror that: peers reach management/signal/relay at this + // name, the proxy registers this as its cluster, and the agent-network + // endpoint is .. + combinedAlias = "netbird.local" + combinedExposedURL = "http://" + combinedAlias + ":8080" + + // containerIssuer is the embedded IdP issuer, used only for internal JWT + // validation (peers authenticate with setup keys / proxy tokens, not OIDC), + // so the in-container localhost address is fine. + containerIssuer = "http://localhost:8080/oauth2" +) + +// Combined is a running combined NetBird server (management + signal + relay + +// STUN + embedded IdP) plus the connection details tests need. It owns the +// shared docker network that the proxy and client containers join. +type Combined struct { + container testcontainers.Container + network *testcontainers.DockerNetwork + // BaseURL is the host-reachable management API root, e.g. http://127.0.0.1:51234. + BaseURL string + // PAT is the admin Personal Access Token minted via Bootstrap. + PAT string + + api *rest.Client + workDir string +} + +// StartCombined builds the combined server from its multistage Dockerfile and +// boots it with setup-PAT enabled on a fresh shared network, returning once the +// API is serving. The caller still owns minting the admin PAT via Bootstrap. +func StartCombined(ctx context.Context) (*Combined, error) { + root, err := repoRoot() + if err != nil { + return nil, err + } + + combinedImage, err := resolveImage(ctx, root, "NB_E2E_COMBINED_IMAGE", defaultCombinedImage, combinedDockerfile) + if err != nil { + return nil, err + } + + net, err := network.New(ctx) + if err != nil { + return nil, fmt.Errorf("create shared network: %w", err) + } + + // Work dir under /tmp so Docker Desktop file sharing (which excludes + // macOS's /var/folders TMPDIR) can bind-mount it. + workDir, err := os.MkdirTemp("/tmp", "nb-e2e-combined-*") + if err != nil { + _ = net.Remove(ctx) + return nil, fmt.Errorf("create work dir: %w", err) + } + + cfg := fmt.Sprintf(combinedConfigYAML, combinedExposedURL, containerIssuer) + if err := os.WriteFile(filepath.Join(workDir, "config.yaml"), []byte(cfg), 0o644); err != nil { //nolint:gosec // non-secret config, bind-mounted and read by the container + _ = net.Remove(ctx) + return nil, fmt.Errorf("write combined config: %w", err) + } + if err := os.MkdirAll(filepath.Join(workDir, "data"), 0o755); err != nil { + _ = net.Remove(ctx) + return nil, fmt.Errorf("create datadir: %w", err) + } + + req := testcontainers.ContainerRequest{ + Image: combinedImage, + ExposedPorts: []string{combinedHTTPPort}, + Networks: []string{net.Name}, + NetworkAliases: map[string][]string{net.Name: {combinedAlias}}, + Env: map[string]string{ + "NB_SETUP_PAT_ENABLED": "true", + // Skip the GeoLite DB download — it blocks startup and agent-network + // ingest doesn't use geolocation. + "NB_DISABLE_GEOLOCATION": "true", + }, + Cmd: []string{"--config", "/nb/config.yaml"}, + HostConfigModifier: func(hc *container.HostConfig) { + hc.Binds = append(hc.Binds, workDir+":/nb") + }, + WaitingFor: wait.ForHTTP("/api/instance"). + WithPort(combinedHTTPPort). + WithStatusCodeMatcher(func(status int) bool { return status == 200 }). + WithStartupTimeout(120 * time.Second), + } + + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + _ = net.Remove(ctx) + return nil, fmt.Errorf("start combined container: %w", err) + } + + host, err := c.Host(ctx) + if err != nil { + _ = c.Terminate(ctx) + _ = net.Remove(ctx) + return nil, fmt.Errorf("container host: %w", err) + } + mapped, err := c.MappedPort(ctx, nat.Port(combinedHTTPPort)) + if err != nil { + _ = c.Terminate(ctx) + _ = net.Remove(ctx) + return nil, fmt.Errorf("mapped port: %w", err) + } + + return &Combined{ + container: c, + network: net, + BaseURL: fmt.Sprintf("http://%s:%s", host, mapped.Port()), + workDir: workDir, + }, nil +} + +// resolveImage returns the image to run for a component. By default it builds +// the image from the repo Dockerfile under localTag, so the e2e exercises the +// branch's code. The env override changes this: a value containing a "/" is a +// registry reference that testcontainers pulls (e.g. to test a published +// release); a bare tag is built under that name instead. +func resolveImage(ctx context.Context, root, envKey, localTag, dockerfile string) (string, error) { + if v := os.Getenv(envKey); v != "" { + if strings.Contains(v, "/") { + return v, nil + } + localTag = v + } + if err := buildImage(ctx, root, dockerfile, localTag); err != nil { + return "", err + } + return localTag, nil +} + +// buildImage builds an image from a repo Dockerfile via buildx with BuildKit, so +// the Dockerfile cache mounts are honored and unchanged layers are reused. The +// result is loaded into the docker image store so testcontainers runs it by tag. +// When NB_E2E_BUILDX_CACHE names a directory (CI, with a container-driver +// builder from docker/setup-buildx-action), layer cache is read from and written +// to it as a local cache so actions/cache can persist it across runs; the Go +// compile itself still re-runs, as BuildKit mount caches can't be exported. +func buildImage(ctx context.Context, root, dockerfile, tag string) error { + args := []string{"buildx", "build", "-f", dockerfile, "-t", tag, "--load"} + if dir := os.Getenv("NB_E2E_BUILDX_CACHE"); dir != "" { + args = append(args, + "--cache-from", "type=local,src="+dir, + "--cache-to", "type=local,dest="+dir+",mode=max", + ) + } + args = append(args, ".") + + cmd := exec.CommandContext(ctx, "docker", args...) + cmd.Dir = root + cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1") + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("build image %s: %w\n%s", tag, err, string(out)) + } + return nil +} + +// CreateProxyTokenCLI mints a proxy access token via the server's `token +// create` CLI inside the container — the same path the manual install uses. +// This yields a GLOBAL (account-less) token, so the proxy serves the whole +// cluster (SynthesizeServicesForCluster); an account-scoped REST token instead +// drives the per-account path. Returns the plaintext token. +func (c *Combined) CreateProxyTokenCLI(ctx context.Context, name string) (string, error) { + code, reader, err := c.container.Exec(ctx, + []string{"/go/bin/netbird-server", "token", "create", "--name", name, "--config", "/nb/config.yaml"}, + tcexec.Multiplexed()) + if err != nil { + return "", fmt.Errorf("exec token create: %w", err) + } + out, _ := io.ReadAll(reader) + if code != 0 { + return "", fmt.Errorf("token create exited %d: %s", code, string(out)) + } + for _, line := range strings.Split(string(out), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "Token:") { + tok := strings.TrimSpace(strings.TrimPrefix(line, "Token:")) + if tok != "" { + return tok, nil + } + } + } + return "", fmt.Errorf("token not found in CLI output: %s", string(out)) +} + +// Logs returns the combined server container logs, for diagnostics. +func (c *Combined) Logs(ctx context.Context) string { + return containerLogs(ctx, c.container) +} + +// Terminate stops the container, removes the shared network, and cleans the +// work dir. +func (c *Combined) Terminate(ctx context.Context) error { + var err error + if c.container != nil { + err = c.container.Terminate(ctx) + } + if c.network != nil { + _ = c.network.Remove(ctx) + } + if c.workDir != "" { + _ = os.RemoveAll(c.workDir) + } + return err +} diff --git a/e2e/harness/config.go b/e2e/harness/config.go new file mode 100644 index 000000000..b4bed60a2 --- /dev/null +++ b/e2e/harness/config.go @@ -0,0 +1,26 @@ +//go:build e2e + +package harness + +// combinedConfigYAML is a minimal combined-server config for tests: plain HTTP +// on :8080 (no TLS cert configured → the server serves HTTP and expects to sit +// behind a reverse proxy, which is exactly what we want for in-cluster tests), +// embedded IdP, local signal/relay/STUN, and a sqlite store under the mounted +// data dir. exposedAddress is the address peers use to reach this container; it +// is overridden per-run so the value matches the container's network alias. +const combinedConfigYAML = `server: + listenAddress: ":8080" + exposedAddress: "%s" + healthcheckAddress: ":9000" + metricsPort: 9090 + logLevel: "info" + logFile: "console" + authSecret: "e2e-relay-secret" + dataDir: "/nb/data" + disableAnonymousMetrics: true + disableGeoliteUpdate: true + auth: + issuer: "%s" + store: + engine: "sqlite" +` diff --git a/e2e/harness/doc.go b/e2e/harness/doc.go new file mode 100644 index 000000000..937d8e664 --- /dev/null +++ b/e2e/harness/doc.go @@ -0,0 +1,13 @@ +//go:build e2e + +// Package harness provides a self-contained, OIDC-free way to stand up NetBird +// components in containers for end-to-end tests. It is feature-agnostic: any +// suite can ask for a live management server (with an admin PAT minted through +// the unauthenticated /api/setup bootstrap) and, later, a proxy and client. +// +// The harness compiles each component once in a cached builder container and +// mounts the resulting binary into a slim runtime container, so iterating on a +// branch doesn't pay a full image rebuild per run. Everything is gated behind +// the `e2e` build tag so normal builds and unit tests never pull in +// testcontainers. +package harness diff --git a/e2e/harness/paths.go b/e2e/harness/paths.go new file mode 100644 index 000000000..d7df6bbfa --- /dev/null +++ b/e2e/harness/paths.go @@ -0,0 +1,29 @@ +//go:build e2e + +package harness + +import ( + "fmt" + "os" + "path/filepath" +) + +// repoRoot walks up from the working directory to the module root (the +// directory holding go.mod), so the Docker build context is correct no matter +// which package the test runs from. +func repoRoot() (string, error) { + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + if _, statErr := os.Stat(filepath.Join(dir, "go.mod")); statErr == nil { + return dir, nil + } + parent := filepath.Dir(dir) + if parent == dir { + return "", fmt.Errorf("go.mod not found above %s", dir) + } + dir = parent + } +} diff --git a/e2e/harness/proxy.go b/e2e/harness/proxy.go new file mode 100644 index 000000000..8db2c140f --- /dev/null +++ b/e2e/harness/proxy.go @@ -0,0 +1,122 @@ +//go:build e2e + +package harness + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +const ( + proxyDockerfile = "proxy/Dockerfile.multistage" + // defaultProxyImage is the local tag the reverse proxy is built under from + // proxyDockerfile. Override with NB_E2E_PROXY_IMAGE: a value with a "/" is + // pulled as a published image; a bare tag is built under that name. + defaultProxyImage = "netbird-reverse-proxy:e2e" + proxyAlias = "proxy" + + // AgentNetworkCluster is the proxy cluster the e2e provider bootstraps and + // the proxy serves. It must equal the management's exposed domain + // (combinedAlias) — the working manual setup uses one NETBIRD_DOMAIN for + // both. The agent-network endpoint is .. + AgentNetworkCluster = combinedAlias +) + +// Proxy is a running agent-network gateway (netbird proxy) container. +type Proxy struct { + container testcontainers.Container + workDir string +} + +// StartProxy builds the proxy image and runs it on the combined server's +// network, registered via the given account proxy token and serving the +// AgentNetworkCluster over a self-signed wildcard cert. It does not wait for +// peer connectivity — callers poll management for the proxy peer. +func StartProxy(ctx context.Context, c *Combined, proxyToken string) (*Proxy, error) { + root, err := repoRoot() + if err != nil { + return nil, err + } + proxyImage, err := resolveImage(ctx, root, "NB_E2E_PROXY_IMAGE", defaultProxyImage, proxyDockerfile) + if err != nil { + return nil, err + } + + workDir, err := os.MkdirTemp("/tmp", "nb-e2e-proxy-*") + if err != nil { + return nil, fmt.Errorf("create proxy work dir: %w", err) + } + // MkdirTemp creates the dir 0700; widen it so the non-root proxy container + // can traverse the bind-mounted cert dir on Linux CI runners. + if err := os.Chmod(workDir, 0o755); err != nil { //nolint:gosec // throwaway e2e cert dir, must be traversable by the proxy container uid + return nil, fmt.Errorf("chmod proxy cert dir: %w", err) + } + if err := writeSelfSignedCert(workDir, []string{"*." + AgentNetworkCluster, AgentNetworkCluster}); err != nil { + return nil, err + } + + req := testcontainers.ContainerRequest{ + Image: proxyImage, + Networks: []string{c.network.Name}, + NetworkAliases: map[string][]string{c.network.Name: {proxyAlias}}, + Env: map[string]string{ + "NB_PROXY_TOKEN": proxyToken, + "NB_PROXY_MANAGEMENT_ADDRESS": combinedExposedURL, + "NB_PROXY_DOMAIN": AgentNetworkCluster, + "NB_PROXY_ADDRESS": ":443", + "NB_PROXY_CERTIFICATE_DIRECTORY": "/certs", + "NB_PROXY_HEALTH_ADDRESS": ":8081", + "NB_PROXY_LOG_LEVEL": "debug", + "NB_PROXY_PRIVATE": "true", + // Management is plain HTTP in-cluster, so allow the proxy token to + // ride a non-TLS gRPC connection. + "NB_PROXY_ALLOW_INSECURE": "true", + // The combined server multiplexes the relay over WebSocket on :8080 + // (no QUIC listener). The proxy's embedded relay client defaults to + // QUIC, which fails here and flaps the relay link, churning the + // proxy peer so it never stably registers. Force WS transport. + "NB_RELAY_TRANSPORT": "ws", + // Trace the embedded client (relay / signal / handshake) so + // peer-registration issues are visible in the proxy logs. + "NB_PROXY_CLIENT_LOG_LEVEL": "trace", + }, + HostConfigModifier: func(hc *container.HostConfig) { + hc.Binds = append(hc.Binds, workDir+":/certs") + hc.CapAdd = append(hc.CapAdd, "NET_ADMIN", "SYS_ADMIN", "SYS_RESOURCE", "NET_BIND_SERVICE") + }, + WaitingFor: wait.ForLog("Initial mapping sync complete").WithStartupTimeout(90 * time.Second), + } + + ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + return nil, fmt.Errorf("start proxy container: %w", err) + } + + return &Proxy{container: ctr, workDir: workDir}, nil +} + +// Logs returns the proxy container logs, for diagnostics on failure. +func (p *Proxy) Logs(ctx context.Context) string { + return containerLogs(ctx, p.container) +} + +// Terminate stops the proxy container and cleans its work dir. +func (p *Proxy) Terminate(ctx context.Context) error { + var err error + if p.container != nil { + err = p.container.Terminate(ctx) + } + if p.workDir != "" { + _ = os.RemoveAll(p.workDir) + } + return err +} diff --git a/go.mod b/go.mod index 047a1bf3d..e1c762607 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,8 @@ require ( github.com/crowdsecurity/go-cs-bouncer v0.0.21 github.com/dexidp/dex v2.13.0+incompatible github.com/dexidp/dex/api/v2 v2.4.0 + github.com/docker/docker v28.0.1+incompatible + github.com/docker/go-connections v0.6.0 github.com/ebitengine/purego v0.8.4 github.com/eko/gocache/lib/v4 v4.2.0 github.com/eko/gocache/store/go_cache/v4 v4.2.2 @@ -188,8 +190,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v28.0.1+incompatible // indirect - github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fredbi/uri v1.1.1 // indirect