[e2e] Add container-based agent-network e2e harness (Pillar 1)

Introduce a self-contained, OIDC-free e2e harness that stands up NetBird
in containers, so suites no longer depend on the hand-maintained Tilt
stack or a real IdP.

- harness brings up the combined server (management + signal + relay +
  STUN + embedded IdP) in a single container built from
  combined/Dockerfile.multistage, and mints an admin PAT through the
  unauthenticated /api/setup bootstrap (NB_SETUP_PAT_ENABLED). API access
  goes through the existing shared/management/client/rest typed client.
- the image is built via the docker CLI (BuildKit) so the Dockerfile's
  cache mounts are honored; testcontainers then runs the tagged image.
- everything is behind the `e2e` build tag so normal builds and unit
  tests never pull in testcontainers.

Adds BuildKit cache mounts to combined/Dockerfile.multistage so source
changes recompile incrementally rather than from scratch.

Pillar 1 proven by TestCombinedBootstrap: server builds, boots, mints a
PAT, and the PAT authenticates a real management API call.
This commit is contained in:
mlsmaycon
2026-06-28 18:39:20 +02:00
parent 6e458d2afe
commit 4a296c5338
9 changed files with 581 additions and 3 deletions

View File

@@ -0,0 +1,259 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"refresh": "",
"schemaVersion": 39,
"tags": [
"netbird",
"sync"
],
"templating": {
"list": [
{
"current": {
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "influxdb",
"uid": "influxdb"
},
"definition": "import \"influxdata/influxdb/schema\"\nschema.tagValues(bucket: \"metrics\", tag: \"version\")",
"includeAll": true,
"label": "version",
"multi": true,
"name": "version",
"query": "import \"influxdata/influxdb/schema\"\nschema.tagValues(bucket: \"metrics\", tag: \"version\")",
"refresh": 2,
"type": "query",
"allValue": ".*"
}
]
},
"time": {
"from": "now-2d",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "NetBird Sync Phases (where time goes)",
"uid": "netbird-sync-phases",
"version": 1,
"panels": [
{
"id": 1,
"title": "Time per phase over time (stacked, ms)",
"type": "timeseries",
"datasource": {
"type": "influxdb",
"uid": "influxdb"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 0
},
"fieldConfig": {
"defaults": {
"unit": "ms",
"custom": {
"drawStyle": "bars",
"stacking": {
"mode": "normal",
"group": "A"
},
"fillOpacity": 80,
"lineWidth": 0
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"calcs": [
"max",
"mean"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "influxdb",
"uid": "influxdb"
},
"query": "from(bucket: \"metrics\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"netbird_sync_phase\" and r._field == \"duration_seconds\")\n |> filter(fn: (r) => r.version =~ /${version:regex}/)\n |> map(fn: (r) => ({ r with _value: r._value * 1000.0 }))\n |> keep(columns: [\"_time\", \"_value\", \"phase\"])\n |> group(columns: [\"phase\"])"
}
]
},
{
"id": 2,
"title": "p95 per phase (ms)",
"type": "bargauge",
"datasource": {
"type": "influxdb",
"uid": "influxdb"
},
"gridPos": {
"h": 11,
"w": 12,
"x": 0,
"y": 10
},
"fieldConfig": {
"defaults": {
"unit": "ms",
"color": {
"mode": "continuous-GrYlRd"
}
},
"overrides": []
},
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": true
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "influxdb",
"uid": "influxdb"
},
"query": "from(bucket: \"metrics\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"netbird_sync_phase\" and r._field == \"duration_seconds\")\n |> filter(fn: (r) => r.version =~ /${version:regex}/)\n |> map(fn: (r) => ({ r with _value: r._value * 1000.0 }))\n |> group(columns: [\"phase\"])\n |> quantile(q: 0.95)\n |> group()\n |> sort(columns: [\"_value\"], desc: true)"
}
]
},
{
"id": 3,
"title": "Per-phase stats (ms): mean / p95 / max",
"type": "table",
"datasource": {
"type": "influxdb",
"uid": "influxdb"
},
"gridPos": {
"h": 11,
"w": 12,
"x": 12,
"y": 10
},
"fieldConfig": {
"defaults": {
"unit": "ms"
},
"overrides": []
},
"options": {
"showHeader": true,
"sortBy": [
{
"displayName": "max",
"desc": true
}
]
},
"transformations": [
{
"id": "merge",
"options": {}
}
],
"targets": [
{
"refId": "mean",
"datasource": {
"type": "influxdb",
"uid": "influxdb"
},
"query": "from(bucket: \"metrics\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"netbird_sync_phase\" and r._field == \"duration_seconds\")\n |> filter(fn: (r) => r.version =~ /${version:regex}/)\n |> map(fn: (r) => ({ r with _value: r._value * 1000.0 }))\n |> group(columns: [\"phase\"])\n |> mean()\n |> group()\n |> keep(columns: [\"phase\", \"_value\"])\n |> rename(columns: {_value: \"mean\"})"
},
{
"refId": "p95",
"datasource": {
"type": "influxdb",
"uid": "influxdb"
},
"query": "from(bucket: \"metrics\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"netbird_sync_phase\" and r._field == \"duration_seconds\")\n |> filter(fn: (r) => r.version =~ /${version:regex}/)\n |> map(fn: (r) => ({ r with _value: r._value * 1000.0 }))\n |> group(columns: [\"phase\"])\n |> quantile(q: 0.95)\n |> group()\n |> keep(columns: [\"phase\", \"_value\"])\n |> rename(columns: {_value: \"p95\"})"
},
{
"refId": "max",
"datasource": {
"type": "influxdb",
"uid": "influxdb"
},
"query": "from(bucket: \"metrics\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"netbird_sync_phase\" and r._field == \"duration_seconds\")\n |> filter(fn: (r) => r.version =~ /${version:regex}/)\n |> map(fn: (r) => ({ r with _value: r._value * 1000.0 }))\n |> group(columns: [\"phase\"])\n |> max()\n |> group()\n |> keep(columns: [\"phase\", \"_value\"])\n |> rename(columns: {_value: \"max\"})"
}
]
},
{
"id": 4,
"title": "Total sync duration (netbird_sync, ms) \u2014 reference",
"type": "timeseries",
"datasource": {
"type": "influxdb",
"uid": "influxdb"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 21
},
"fieldConfig": {
"defaults": {
"unit": "ms",
"custom": {
"drawStyle": "points",
"pointSize": 5
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "right",
"calcs": [
"max",
"mean"
]
},
"tooltip": {
"mode": "single"
}
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "influxdb",
"uid": "influxdb"
},
"query": "from(bucket: \"metrics\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"netbird_sync\" and r._field == \"duration_seconds\")\n |> filter(fn: (r) => r.version =~ /${version:regex}/)\n |> map(fn: (r) => ({ r with _value: r._value * 1000.0 }))\n |> keep(columns: [\"_time\", \"_value\", \"version\"])\n |> group(columns: [\"version\"])"
}
]
}
]
}

View File

@@ -59,6 +59,19 @@ var allowedMeasurements = map[string]measurementSpec{
"peer_id": true,
},
},
"netbird_sync_phase": {
allowedFields: map[string]bool{
"duration_seconds": true,
},
allowedTags: map[string]bool{
"deployment_type": true,
"version": true,
"os": true,
"arch": true,
"peer_id": true,
"phase": true,
},
},
"netbird_login": {
allowedFields: map[string]bool{
"duration_seconds": true,

View File

@@ -5,12 +5,16 @@ WORKDIR /app
RUN apt-get update && apt-get install -y gcc libc6-dev git && rm -rf /var/lib/apt/lists/*
COPY go.mod go.sum ./
RUN go mod download
RUN --mount=type=cache,target=/go/pkg/mod go mod download
COPY . .
# Build with version info from git (matching goreleaser ldflags)
RUN CGO_ENABLED=1 GOOS=linux go build \
# Build with version info from git (matching goreleaser ldflags).
# BuildKit cache mounts persist the module + build caches across image builds,
# so a source change recompiles incrementally instead of from scratch.
RUN --mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
CGO_ENABLED=1 GOOS=linux go build \
-ldflags="-s -w \
-X github.com/netbirdio/netbird/version.version=$(git describe --tags --always --dirty 2>/dev/null || echo 'dev') \
-X main.commit=$(git rev-parse --short HEAD 2>/dev/null || echo 'unknown') \

View File

@@ -0,0 +1,43 @@
//go:build e2e
package agentnetwork
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/netbirdio/netbird/e2e/harness"
)
// TestCombinedBootstrap proves Pillar 1: a combined NetBird server comes up in a
// container, the /api/setup bootstrap mints an admin PAT with no OIDC, and that
// PAT authenticates real management API calls through the typed REST client.
func TestCombinedBootstrap(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
srv, err := harness.StartCombined(ctx)
require.NoError(t, err, "combined server must build and start")
t.Cleanup(func() {
_ = srv.Terminate(context.Background())
})
pat, err := srv.Bootstrap(ctx)
require.NoError(t, err, "/api/setup must mint an admin PAT")
require.NotEmpty(t, pat, "PAT must be non-empty")
// The PAT must authenticate a real management API call through the client.
users, err := srv.API().Users.List(ctx)
require.NoError(t, err, "authenticated Users.List must round-trip")
require.NotEmpty(t, users, "the bootstrapped account must have at least one user")
var emails []string
for _, u := range users {
emails = append(emails, u.Email)
}
assert.Contains(t, emails, "admin@netbird.test", "the bootstrapped owner should appear in the users list")
}

47
e2e/harness/bootstrap.go Normal file
View File

@@ -0,0 +1,47 @@
//go:build e2e
package harness
import (
"context"
"fmt"
"github.com/netbirdio/netbird/shared/management/client/rest"
"github.com/netbirdio/netbird/shared/management/http/api"
)
// Bootstrap creates the initial admin owner through the unauthenticated
// /api/setup endpoint and returns the plaintext admin PAT. It also wires an
// authenticated REST client on the Combined (see API). create_pat requires the
// server to run with NB_SETUP_PAT_ENABLED=true, which the harness sets. A
// second call returns an error (the server reports setup already completed).
func (c *Combined) Bootstrap(ctx context.Context) (string, error) {
// The setup endpoint is unauthenticated; use a tokenless client.
setupClient := rest.NewWithOptions(rest.WithManagementURL(c.BaseURL))
createPAT := true
expireDays := 1
resp, err := setupClient.Instance.Setup(ctx, api.PostApiSetupJSONRequestBody{
Email: "admin@netbird.test",
Password: "Netbird-e2e-Passw0rd!",
Name: "E2E Admin",
CreatePat: &createPAT,
PatExpireIn: &expireDays,
})
if err != nil {
return "", fmt.Errorf("instance setup: %w", err)
}
if resp.PersonalAccessToken == nil || *resp.PersonalAccessToken == "" {
return "", fmt.Errorf("setup succeeded but no PAT returned (is NB_SETUP_PAT_ENABLED set?)")
}
c.PAT = *resp.PersonalAccessToken
c.api = rest.New(c.BaseURL, c.PAT)
return c.PAT, nil
}
// API returns the PAT-authenticated management REST client. It is nil until
// Bootstrap runs.
func (c *Combined) API() *rest.Client {
return c.api
}

146
e2e/harness/combined.go Normal file
View File

@@ -0,0 +1,146 @@
//go:build e2e
package harness
import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"time"
"github.com/docker/docker/api/types/container"
"github.com/docker/go-connections/nat"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/wait"
"github.com/netbirdio/netbird/shared/management/client/rest"
)
const (
combinedDockerfile = "combined/Dockerfile.multistage"
combinedImageTag = "netbird-e2e-combined:latest"
combinedHTTPPort = "8080/tcp"
// containerInternalURL is how the server addresses itself: it listens on
// :8080 inside the container, so the embedded IdP issuer and exposed
// address both resolve there. Tests reach it via the mapped host port.
containerInternalURL = "http://localhost:8080"
containerIssuer = containerInternalURL + "/oauth2"
)
// Combined is a running combined NetBird server (management + signal + relay +
// STUN + embedded IdP) plus the connection details tests need.
type Combined struct {
container testcontainers.Container
// BaseURL is the host-reachable management API root, e.g. http://127.0.0.1:51234.
BaseURL string
// PAT is the admin Personal Access Token minted via Bootstrap.
PAT string
api *rest.Client
workDir string
}
// StartCombined builds the combined server from its multistage Dockerfile and
// boots it with setup-PAT enabled, returning once the API is serving. The
// caller still owns minting the admin PAT via Bootstrap.
func StartCombined(ctx context.Context) (*Combined, error) {
root, err := repoRoot()
if err != nil {
return nil, err
}
// Build with the docker CLI (BuildKit) rather than testcontainers' classic
// build path so the Dockerfile's cache mounts are honored and recompiles
// stay incremental. testcontainers then just runs the tagged image.
if err := buildCombinedImage(ctx, root); err != nil {
return nil, err
}
// Work dir under /tmp so Docker Desktop file sharing (which excludes
// macOS's /var/folders TMPDIR) can bind-mount it.
workDir, err := os.MkdirTemp("/tmp", "nb-e2e-combined-*")
if err != nil {
return nil, fmt.Errorf("create work dir: %w", err)
}
cfg := fmt.Sprintf(combinedConfigYAML, containerInternalURL, containerIssuer)
if err := os.WriteFile(filepath.Join(workDir, "config.yaml"), []byte(cfg), 0o644); err != nil {
return nil, fmt.Errorf("write combined config: %w", err)
}
if err := os.MkdirAll(filepath.Join(workDir, "data"), 0o755); err != nil {
return nil, fmt.Errorf("create datadir: %w", err)
}
req := testcontainers.ContainerRequest{
Image: combinedImageTag,
ExposedPorts: []string{combinedHTTPPort},
Env: map[string]string{
"NB_SETUP_PAT_ENABLED": "true",
},
Cmd: []string{"--config", "/nb/config.yaml"},
HostConfigModifier: func(hc *container.HostConfig) {
hc.Binds = append(hc.Binds, workDir+":/nb")
},
WaitingFor: wait.ForHTTP("/api/instance").
WithPort(combinedHTTPPort).
WithStatusCodeMatcher(func(status int) bool { return status == 200 }).
WithStartupTimeout(90 * time.Second),
}
c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: req,
Started: true,
})
if err != nil {
return nil, fmt.Errorf("start combined container: %w", err)
}
host, err := c.Host(ctx)
if err != nil {
_ = c.Terminate(ctx)
return nil, fmt.Errorf("container host: %w", err)
}
mapped, err := c.MappedPort(ctx, nat.Port(combinedHTTPPort))
if err != nil {
_ = c.Terminate(ctx)
return nil, fmt.Errorf("mapped port: %w", err)
}
return &Combined{
container: c,
BaseURL: fmt.Sprintf("http://%s:%s", host, mapped.Port()),
workDir: workDir,
}, nil
}
// buildCombinedImage builds the combined server image via the docker CLI with
// BuildKit enabled, so the Dockerfile's cache mounts work and unchanged sources
// reuse the layer + go caches. The image is tagged stably so reruns are cheap.
func buildCombinedImage(ctx context.Context, root string) error {
cmd := exec.CommandContext(ctx, "docker", "build",
"-f", combinedDockerfile,
"-t", combinedImageTag,
".",
)
cmd.Dir = root
cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1")
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("build combined image: %w\n%s", err, string(out))
}
return nil
}
// Terminate stops the container and removes the work dir.
func (c *Combined) Terminate(ctx context.Context) error {
var err error
if c.container != nil {
err = c.container.Terminate(ctx)
}
if c.workDir != "" {
_ = os.RemoveAll(c.workDir)
}
return err
}

24
e2e/harness/config.go Normal file
View File

@@ -0,0 +1,24 @@
//go:build e2e
package harness
// combinedConfigYAML is a minimal combined-server config for tests: plain HTTP
// on :8080 (no TLS cert configured → the server serves HTTP and expects to sit
// behind a reverse proxy, which is exactly what we want for in-cluster tests),
// embedded IdP, local signal/relay/STUN, and a sqlite store under the mounted
// data dir. exposedAddress is the address peers use to reach this container; it
// is overridden per-run so the value matches the container's network alias.
const combinedConfigYAML = `server:
listenAddress: ":8080"
exposedAddress: "%s"
healthcheckAddress: ":9000"
metricsPort: 9090
logLevel: "info"
logFile: "console"
authSecret: "e2e-relay-secret"
dataDir: "/nb/data"
auth:
issuer: "%s"
store:
engine: "sqlite"
`

13
e2e/harness/doc.go Normal file
View File

@@ -0,0 +1,13 @@
//go:build e2e
// Package harness provides a self-contained, OIDC-free way to stand up NetBird
// components in containers for end-to-end tests. It is feature-agnostic: any
// suite can ask for a live management server (with an admin PAT minted through
// the unauthenticated /api/setup bootstrap) and, later, a proxy and client.
//
// The harness compiles each component once in a cached builder container and
// mounts the resulting binary into a slim runtime container, so iterating on a
// branch doesn't pay a full image rebuild per run. Everything is gated behind
// the `e2e` build tag so normal builds and unit tests never pull in
// testcontainers.
package harness

29
e2e/harness/paths.go Normal file
View File

@@ -0,0 +1,29 @@
//go:build e2e
package harness
import (
"fmt"
"os"
"path/filepath"
)
// repoRoot walks up from the working directory to the module root (the
// directory holding go.mod), so the Docker build context is correct no matter
// which package the test runs from.
func repoRoot() (string, error) {
dir, err := os.Getwd()
if err != nil {
return "", err
}
for {
if _, statErr := os.Stat(filepath.Join(dir, "go.mod")); statErr == nil {
return dir, nil
}
parent := filepath.Dir(dir)
if parent == dir {
return "", fmt.Errorf("go.mod not found above %s", dir)
}
dir = parent
}
}