sendnrw/netbird
2
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2026-07-16 11:39:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

[e2e] Add live chat-through-proxy scenario (Pillar 3)

Browse Source 
Stand up the full agent-network data path in containers and drive a real
chat-completion through the gateway:

- harness: a shared docker network (combined server reachable by alias),
  a proxy container built from the published reverse-proxy image
  (NB_PROXY_PRIVATE, NB_PROXY_ALLOW_INSECURE, NB_RELAY_TRANSPORT=ws to match
  the combined server's WS-multiplexed relay) with a generated self-signed
  wildcard cert, and a netbird client container that joins via a setup key.
- the combined image, proxy image, and client image default to the
  published rc.2 releases (overridable via NB_E2E_*_IMAGE; a bare local tag
  is built from source instead). Geolocation download is disabled so the
  server starts without external fetches.
- one shared domain is used for the management exposed address, the proxy
  domain, and the agent-network cluster; the proxy token is minted via the
  server CLI (global) to match the manual install.

TestChatCompletionThroughProxy provisions provider+policy+group+setup key,
runs proxy+client, drives an OpenAI chat-completion through the tunnel, and
asserts a 200 plus the ingested access-log row. Requires OPENAI_TOKEN
(skips otherwise). The provider must be created with enabled=true explicitly
— the create default is false despite the API doc.
This commit is contained in:
mlsmaycon
2026-06-29 10:34:43 +02:00
parent d9d437339c
commit 3e6aba7dfe
9 changed files with 705 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
e2e/agentnetwork/chat_test.go Normal file
View File

@@ -0,0 +1,165 @@
//go:build e2e
package agentnetwork
import (
"context"
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/netbirdio/netbird/e2e/harness"
"github.com/netbirdio/netbird/shared/management/http/api"
)
// TestChatCompletionThroughProxy is Pillar 3: it provisions an agent-network
// gateway (provider + policy + setup key), runs the proxy and a client
// container on the shared network, and drives a real chat-completion from the
// client through the proxy to the upstream provider over the WireGuard tunnel,
// asserting a 200 and that usage is recorded.
//
// Requires a real provider key in OPENAI_TOKEN (source ~/.llm-keys locally; set
// the Actions secret in CI). Skips otherwise.
func TestChatCompletionThroughProxy(t *testing.T) {
apiKey := os.Getenv("OPENAI_TOKEN")
if apiKey == "" {
t.Skip("OPENAI_TOKEN not set; source ~/.llm-keys to run the live chat test")
}
ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
defer cancel()
// Group + setup key: the client joins into this group; the policy authorizes
// it to reach the provider.
grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-agents"})
require.NoError(t, err, "create agents group")
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
ephemeral := false
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
Name: "e2e-client",
Type: "reusable",
ExpiresIn: 86400,
UsageLimit: 0,
AutoGroups: []string{grp.Id},
Ephemeral: &ephemeral,
})
require.NoError(t, err, "mint setup key")
require.NotEmpty(t, sk.Key, "setup key must be returned in plaintext")
// Provider (real upstream key) + policy authorizing the group. Created
// before the proxy starts so the proxy's initial cluster snapshot already
// carries the account's synthesized service.
prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
Name: "OpenAI Live",
ProviderId: "openai_api",
UpstreamUrl: "https://api.openai.com",
ApiKey: &apiKey,
Enabled: ptr(true),
BootstrapCluster: ptr(harness.AgentNetworkCluster),
Models: &[]api.AgentNetworkProviderModel{
{Id: "gpt-4o-mini", InputPer1k: 0.00015, OutputPer1k: 0.0006},
},
})
require.NoError(t, err, "create provider")
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
enabled := true
policyReq := api.AgentNetworkPolicyRequest{
Name: "e2e-allow",
Enabled: &enabled,
SourceGroups: []string{grp.Id},
DestinationProviderIds: []string{prov.Id},
}
pol, err := srv.CreatePolicy(ctx, policyReq)
require.NoError(t, err, "create policy")
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
settings, err := srv.GetSettings(ctx)
require.NoError(t, err, "read settings for endpoint")
require.NotEmpty(t, settings.Endpoint, "agent-network endpoint must be assigned")
// Mint the proxy token via the server CLI (global, account-less) — the path
// the manual install uses, which drives the cluster-snapshot synthesis the
// proxy needs. An account-scoped REST token takes a different path that
// doesn't deliver the service.
proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-proxy")
require.NoError(t, err, "mint proxy token via CLI")
px, err := harness.StartProxy(ctx, srv, proxyToken)
require.NoError(t, err, "start proxy")
t.Cleanup(func() { _ = px.Terminate(context.Background()) })
// Client joins last, once the proxy + provider + policy are all in place, so
// its initial network map includes the synthesized agent-network service.
cl, err := harness.StartClient(ctx, srv, sk.Key)
require.NoError(t, err, "start client")
t.Cleanup(func() { _ = cl.Terminate(context.Background()) })
require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management")
if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil {
dctx := context.Background()
peers, _ := srv.API().Peers.List(dctx)
var peerInfo []string
for _, p := range peers {
var groups []string
for _, g := range p.Groups {
groups = append(groups, g.Name)
}
peerInfo = append(peerInfo, fmt.Sprintf("%s connected=%t ip=%s groups=%v", p.Name, p.Connected, p.Ip, groups))
}
clusters, _ := srv.API().ReverseProxyClusters.List(dctx)
var clusterInfo []string
for _, cl := range clusters {
clusterInfo = append(clusterInfo, fmt.Sprintf("%+v", cl))
}
domains, _ := srv.API().ReverseProxyDomains.List(dctx)
var domainInfo []string
for _, d := range domains {
domainInfo = append(domainInfo, fmt.Sprintf("%+v", d))
}
_ = os.WriteFile("/tmp/nb-e2e-proxy.log", []byte(px.Logs(dctx)), 0o644)
_ = os.WriteFile("/tmp/nb-e2e-client.log", []byte(cl.Logs(dctx)), 0o644)
_ = os.WriteFile("/tmp/nb-e2e-combined.log", []byte(srv.Logs(dctx)), 0o644)
diag := fmt.Sprintf("settings: cluster=%q endpoint=%q subdomain=%q\nprovider: id=%s cluster=%s\npolicy: id=%s sourceGroups=%v dst=%v\ngroup: id=%s\npeers:\n%s\nclusters:\n%s\n",
settings.Cluster, settings.Endpoint, settings.Subdomain,
prov.Id, harness.AgentNetworkCluster,
pol.Id, policyReq.SourceGroups, policyReq.DestinationProviderIds,
grp.Id,
strings.Join(peerInfo, "\n"), strings.Join(clusterInfo, "\n"))
diag += "domains:\n" + strings.Join(domainInfo, "\n") + "\n"
_ = os.WriteFile("/tmp/nb-e2e-diag.txt", []byte(diag), 0o644)
t.Fatalf("client did not see the proxy peer: %v\n=== settings ===\ncluster=%q endpoint=%q subdomain=%q\n=== peers ===\n%v\n=== clusters ===\n%v\n=== proxy logs ===\n%s",
err, settings.Cluster, settings.Endpoint, settings.Subdomain, peerInfo, clusterInfo, px.Logs(dctx))
}
proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint)
require.NoError(t, err, "resolve agent-network endpoint to proxy IP")
code, body, err := cl.Chat(ctx, settings.Endpoint, proxyIP, "gpt-4o-mini", "Reply with exactly: pong")
require.NoError(t, err, "chat request through tunnel")
if code != 200 {
t.Fatalf("expected 200 from chat-completion, got %d\nbody: %s\n=== proxy logs ===\n%s", code, body, px.Logs(context.Background()))
}
assert.Contains(t, body, "choices", "chat response should carry choices")
// The per-request access-log row is ingested asynchronously after the
// response is forwarded; poll briefly. (Consumption rows are only booked
// when a policy has token/budget limits, which this one doesn't.)
require.Eventually(t, func() bool {
resp, lerr := srv.ListAccessLogs(ctx)
return lerr == nil && resp.TotalRecords > 0
}, 30*time.Second, 2*time.Second, "an access-log row should be recorded after the chat-completion")
logs, err := srv.ListAccessLogs(ctx)
require.NoError(t, err, "read access logs")
require.NotEmpty(t, logs.Data, "access-log page must contain the request row")
require.NotNil(t, logs.Data[0].Model, "access-log row should record the model")
assert.Equal(t, "gpt-4o-mini", *logs.Data[0].Model, "access-log row should record the requested model")
}

24
e2e/harness/Dockerfile.client Normal file
View File

@@ -0,0 +1,24 @@
# Multistage build for the NetBird client used in e2e tests. The repo has no
# source-building client Dockerfile (client/Dockerfile packages a goreleaser
# artifact), so this mirrors its alpine runtime + entrypoint while compiling the
# CGO-free client inline. BuildKit cache mounts keep rebuilds incremental.
FROM golang:1.25-bookworm AS builder
WORKDIR /src
COPY go.mod go.sum ./
RUN --mount=type=cache,target=/go/pkg/mod go mod download
COPY . .
RUN --mount=type=cache,target=/go/pkg/mod \
--mount=type=cache,target=/root/.cache/go-build \
CGO_ENABLED=0 GOOS=linux go build -o /out/netbird ./client
FROM alpine:3.24
RUN apk add --no-cache bash ca-certificates ip6tables iproute2 iptables
ENV NETBIRD_BIN="/usr/local/bin/netbird" \
NB_LOG_FILE="console,/var/log/netbird/client.log" \
NB_DAEMON_ADDR="unix:///var/run/netbird.sock" \
NB_ENABLE_CAPTURE="false" \
NB_ENTRYPOINT_SERVICE_TIMEOUT="30"
ENTRYPOINT [ "/usr/local/bin/netbird-entrypoint.sh" ]
COPY client/netbird-entrypoint.sh /usr/local/bin/netbird-entrypoint.sh
COPY --from=builder /out/netbird /usr/local/bin/netbird

11
e2e/harness/agentnetwork.go
View File

@@ -79,6 +79,11 @@ func (c *Combined) CreatePolicy(ctx context.Context, req api.AgentNetworkPolicyR
return anRequest[api.AgentNetworkPolicy](ctx, c, http.MethodPost, "/api/agent-network/policies", req)
}
// UpdatePolicy replaces a policy by id.
func (c *Combined) UpdatePolicy(ctx context.Context, id string, req api.AgentNetworkPolicyRequest) (api.AgentNetworkPolicy, error) {
return anRequest[api.AgentNetworkPolicy](ctx, c, http.MethodPut, "/api/agent-network/policies/"+id, req)
}
// DeletePolicy removes a policy by id.
func (c *Combined) DeletePolicy(ctx context.Context, id string) error {
return anDelete(ctx, c, "/api/agent-network/policies/"+id)
@@ -99,3 +104,9 @@ func (c *Combined) UpdateSettings(ctx context.Context, req api.AgentNetworkSetti
func (c *Combined) ListConsumption(ctx context.Context) ([]api.AgentNetworkConsumption, error) {
return anRequest[[]api.AgentNetworkConsumption](ctx, c, http.MethodGet, "/api/agent-network/consumption", nil)
}
// ListAccessLogs returns the account's agent-network access-log page (the
// flattened per-request rows the proxy ships and management ingests).
func (c *Combined) ListAccessLogs(ctx context.Context) (api.AgentNetworkAccessLogsResponse, error) {
return anRequest[api.AgentNetworkAccessLogsResponse](ctx, c, http.MethodGet, "/api/agent-network/access-logs", nil)
}

2
e2e/harness/bootstrap.go
View File

@@ -21,7 +21,7 @@ func (c *Combined) Bootstrap(ctx context.Context) (string, error) {
createPAT := true
expireDays := 1
resp, err := setupClient.Instance.Setup(ctx, api.PostApiSetupJSONRequestBody{
resp, err := setupClient.Instance.Setup(ctx, api.PostApiSetupJSONRequestBody{ //nolint:gosec // static throwaway test credentials
Email: "admin@netbird.test",
Password: "Netbird-e2e-Passw0rd!",
Name: "E2E Admin",

64
e2e/harness/cert.go Normal file
View File

@@ -0,0 +1,64 @@
//go:build e2e
package harness
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"
"math/big"
"os"
"path/filepath"
"time"
)
// writeSelfSignedCert generates a self-signed TLS cert/key pair covering the
// given DNS names and writes them as tls.crt / tls.key in dir. The proxy serves
// this for the agent-network endpoint; the client curls with -k, so validity
// chains don't matter — the proxy just needs a usable cert to present.
func writeSelfSignedCert(dir string, dnsNames []string) error {
priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return fmt.Errorf("generate key: %w", err)
}
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
if err != nil {
return fmt.Errorf("generate serial: %w", err)
}
tmpl := x509.Certificate{
SerialNumber: serial,
Subject: pkix.Name{CommonName: dnsNames[0]},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(365 * 24 * time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
DNSNames: dnsNames,
BasicConstraintsValid: true,
}
der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv)
if err != nil {
return fmt.Errorf("create certificate: %w", err)
}
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
if err := os.WriteFile(filepath.Join(dir, "tls.crt"), certPEM, 0o644); err != nil { //nolint:gosec // public cert, bind-mounted and read by the proxy container
return fmt.Errorf("write cert: %w", err)
}
keyDER, err := x509.MarshalECPrivateKey(priv)
if err != nil {
return fmt.Errorf("marshal key: %w", err)
}
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})
if err := os.WriteFile(filepath.Join(dir, "tls.key"), keyPEM, 0o600); err != nil {
return fmt.Errorf("write key: %w", err)
}
return nil
}

207
e2e/harness/client.go Normal file
View File

@@ -0,0 +1,207 @@
//go:build e2e
package harness
import (
"context"
"fmt"
"io"
"os/exec"
"strings"
"time"
"github.com/docker/docker/api/types/container"
"github.com/testcontainers/testcontainers-go"
tcexec "github.com/testcontainers/testcontainers-go/exec"
)
const (
clientDockerfile = "e2e/harness/Dockerfile.client"
// defaultClientImage is the published NetBird client release used by
// default. Override with NB_E2E_CLIENT_IMAGE; a value without a "/" is built
// locally from clientDockerfile.
defaultClientImage = "netbirdio/netbird:0.74.0-rc.2"
clientAlias = "client"
curlImage = "curlimages/curl:latest"
)
// Client is a running NetBird client container joined to the combined server.
type Client struct {
container testcontainers.Container
}
// StartClient builds the client image and runs it on the combined server's
// network, joining via the given setup key. The image entrypoint brings the
// daemon up automatically; callers wait for connectivity with WaitConnected /
// WaitProxyPeer.
func StartClient(ctx context.Context, c *Combined, setupKey string) (*Client, error) {
root, err := repoRoot()
if err != nil {
return nil, err
}
clientImage, err := resolveImage(ctx, root, "NB_E2E_CLIENT_IMAGE", defaultClientImage, clientDockerfile)
if err != nil {
return nil, err
}
req := testcontainers.ContainerRequest{
Image: clientImage,
Networks: []string{c.network.Name},
NetworkAliases: map[string][]string{c.network.Name: {clientAlias}},
Env: map[string]string{
"NB_MANAGEMENT_URL": combinedExposedURL,
"NB_SETUP_KEY": setupKey,
"NB_LOG_LEVEL": "info",
// Match the proxy: the combined relay is WebSocket-only, so the
// client must use WS transport to keep a stable relay link to it.
"NB_RELAY_TRANSPORT": "ws",
},
HostConfigModifier: func(hc *container.HostConfig) {
hc.CapAdd = append(hc.CapAdd, "NET_ADMIN", "SYS_ADMIN", "SYS_RESOURCE")
},
}
ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: req,
Started: true,
})
if err != nil {
return nil, fmt.Errorf("start client container: %w", err)
}
return &Client{container: ctr}, nil
}
// Restart bounces the client connection (netbird down/up) so it pulls a fresh
// network map — the documented workaround for a freshly-joined client not yet
// seeing a synthesized agent-network service.
func (cl *Client) Restart(ctx context.Context) error {
if _, _, err := cl.container.Exec(ctx, []string{"netbird", "down"}, tcexec.Multiplexed()); err != nil {
return fmt.Errorf("netbird down: %w", err)
}
time.Sleep(2 * time.Second)
code, reader, err := cl.container.Exec(ctx, []string{"netbird", "up"}, tcexec.Multiplexed())
if err != nil {
return fmt.Errorf("netbird up: %w", err)
}
if code != 0 {
out, _ := io.ReadAll(reader)
return fmt.Errorf("netbird up exited %d: %s", code, string(out))
}
return nil
}
// Status returns `netbird status` output from inside the client.
func (cl *Client) Status(ctx context.Context) (string, error) {
code, reader, err := cl.container.Exec(ctx, []string{"netbird", "status"}, tcexec.Multiplexed())
if err != nil {
return "", err
}
out, _ := io.ReadAll(reader)
if code != 0 {
return string(out), fmt.Errorf("netbird status exited %d", code)
}
return string(out), nil
}
// WaitConnected polls until the client reports Management: Connected.
func (cl *Client) WaitConnected(ctx context.Context, timeout time.Duration) error {
return cl.pollStatus(ctx, timeout, "Management: Connected")
}
// WaitProxyPeer polls until the client sees the proxy peer connected (1/1).
func (cl *Client) WaitProxyPeer(ctx context.Context, timeout time.Duration) error {
return cl.pollStatus(ctx, timeout, "1/1 Connected")
}
func (cl *Client) pollStatus(ctx context.Context, timeout time.Duration, want string) error {
deadline := time.Now().Add(timeout)
var last string
for time.Now().Before(deadline) {
out, _ := cl.Status(ctx)
last = out
if strings.Contains(out, want) {
return nil
}
time.Sleep(3 * time.Second)
}
return fmt.Errorf("timed out waiting for %q; last status:\n%s", want, last)
}
// ResolveProxyIP resolves the agent-network endpoint to the proxy peer's
// NetBird IP from inside the client (via magic DNS).
func (cl *Client) ResolveProxyIP(ctx context.Context, endpoint string) (string, error) {
code, reader, err := cl.container.Exec(ctx, []string{"getent", "hosts", endpoint}, tcexec.Multiplexed())
if err != nil {
return "", err
}
out, _ := io.ReadAll(reader)
if code != 0 {
return "", fmt.Errorf("getent hosts %s exited %d", endpoint, code)
}
fields := strings.Fields(string(out))
if len(fields) == 0 {
return "", fmt.Errorf("no address for %s", endpoint)
}
return fields[0], nil
}
// Chat issues a chat-completion POST to the agent-network endpoint over the
// client's tunnel, returning the HTTP status and response body. It runs curl in
// a throwaway container sharing the client's network namespace so the request
// traverses the WireGuard tunnel, pinning the endpoint to the proxy peer IP.
func (cl *Client) Chat(ctx context.Context, endpoint, proxyIP, model, prompt string) (int, string, error) {
body := fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":%q}]}`, model, prompt)
url := "https://" + endpoint + "/v1/chat/completions"
args := []string{
"run", "--rm",
"--network", "container:" + cl.container.GetContainerID(),
curlImage,
"-sk", "--connect-timeout", "5", "--max-time", "90",
"--resolve", endpoint + ":443:" + proxyIP,
"-o", "/dev/stderr", "-w", "%{http_code}",
"-X", "POST", url,
"-H", "Content-Type: application/json",
"--data", body,
}
cmd := exec.CommandContext(ctx, "docker", args...)
// -w writes the status code to stdout; -o /dev/stderr writes the body to
// stderr so we can capture both separately.
var stdout, stderr strings.Builder
cmd.Stdout = &stdout
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return 0, stderr.String(), fmt.Errorf("curl through tunnel: %w", err)
}
code := 0
_, _ = fmt.Sscanf(strings.TrimSpace(stdout.String()), "%d", &code)
return code, stderr.String(), nil
}
// Logs returns the client container logs, for diagnostics on failure.
func (cl *Client) Logs(ctx context.Context) string {
return containerLogs(ctx, cl.container)
}
// Terminate stops the client container.
func (cl *Client) Terminate(ctx context.Context) error {
if cl.container == nil {
return nil
}
return cl.container.Terminate(ctx)
}
// containerLogs reads up to 256 KiB of a container's logs for diagnostics.
func containerLogs(ctx context.Context, c testcontainers.Container) string {
if c == nil {
return ""
}
r, err := c.Logs(ctx)
if err != nil {
return fmt.Sprintf("<logs error: %v>", err)
}
defer r.Close()
b, _ := io.ReadAll(io.LimitReader(r, 256<<10))
return string(b)
}

142
e2e/harness/combined.go
View File

@@ -5,14 +5,18 @@ package harness
import (
"context"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/docker/docker/api/types/container"
"github.com/docker/go-connections/nat"
"github.com/testcontainers/testcontainers-go"
tcexec "github.com/testcontainers/testcontainers-go/exec"
"github.com/testcontainers/testcontainers-go/network"
"github.com/testcontainers/testcontainers-go/wait"
"github.com/netbirdio/netbird/shared/management/client/rest"
@@ -20,20 +24,34 @@ import (
const (
combinedDockerfile = "combined/Dockerfile.multistage"
combinedImageTag = "netbird-e2e-combined:latest"
combinedHTTPPort = "8080/tcp"
// defaultCombinedImage is the published combined-server release used by
// default (the artifact operators run manually). Override with
// NB_E2E_COMBINED_IMAGE; a value without a "/" is built locally from
// combinedDockerfile instead of pulled.
defaultCombinedImage = "netbirdio/netbird-server:0.74.0-rc.2"
combinedHTTPPort = "8080/tcp"
// containerInternalURL is how the server addresses itself: it listens on
// :8080 inside the container, so the embedded IdP issuer and exposed
// address both resolve there. Tests reach it via the mapped host port.
containerInternalURL = "http://localhost:8080"
containerIssuer = containerInternalURL + "/oauth2"
// combinedAlias is the combined server's network alias AND the deployment
// domain. The working manual setup uses a single NETBIRD_DOMAIN for the
// management exposed address, the proxy domain, and the agent-network
// cluster — so we mirror that: peers reach management/signal/relay at this
// name, the proxy registers this as its cluster, and the agent-network
// endpoint is <subdomain>.<combinedAlias>.
combinedAlias = "netbird.local"
combinedExposedURL = "http://" + combinedAlias + ":8080"
// containerIssuer is the embedded IdP issuer, used only for internal JWT
// validation (peers authenticate with setup keys / proxy tokens, not OIDC),
// so the in-container localhost address is fine.
containerIssuer = "http://localhost:8080/oauth2"
)
// Combined is a running combined NetBird server (management + signal + relay +
// STUN + embedded IdP) plus the connection details tests need.
// STUN + embedded IdP) plus the connection details tests need. It owns the
// shared docker network that the proxy and client containers join.
type Combined struct {
container testcontainers.Container
network *testcontainers.DockerNetwork
// BaseURL is the host-reachable management API root, e.g. http://127.0.0.1:51234.
BaseURL string
// PAT is the admin Personal Access Token minted via Bootstrap.
@@ -44,41 +62,52 @@ type Combined struct {
}
// StartCombined builds the combined server from its multistage Dockerfile and
// boots it with setup-PAT enabled, returning once the API is serving. The
// caller still owns minting the admin PAT via Bootstrap.
// boots it with setup-PAT enabled on a fresh shared network, returning once the
// API is serving. The caller still owns minting the admin PAT via Bootstrap.
func StartCombined(ctx context.Context) (*Combined, error) {
root, err := repoRoot()
if err != nil {
return nil, err
}
// Build with the docker CLI (BuildKit) rather than testcontainers' classic
// build path so the Dockerfile's cache mounts are honored and recompiles
// stay incremental. testcontainers then just runs the tagged image.
if err := buildCombinedImage(ctx, root); err != nil {
combinedImage, err := resolveImage(ctx, root, "NB_E2E_COMBINED_IMAGE", defaultCombinedImage, combinedDockerfile)
if err != nil {
return nil, err
}
net, err := network.New(ctx)
if err != nil {
return nil, fmt.Errorf("create shared network: %w", err)
}
// Work dir under /tmp so Docker Desktop file sharing (which excludes
// macOS's /var/folders TMPDIR) can bind-mount it.
workDir, err := os.MkdirTemp("/tmp", "nb-e2e-combined-*")
if err != nil {
_ = net.Remove(ctx)
return nil, fmt.Errorf("create work dir: %w", err)
}
cfg := fmt.Sprintf(combinedConfigYAML, containerInternalURL, containerIssuer)
if err := os.WriteFile(filepath.Join(workDir, "config.yaml"), []byte(cfg), 0o644); err != nil {
cfg := fmt.Sprintf(combinedConfigYAML, combinedExposedURL, containerIssuer)
if err := os.WriteFile(filepath.Join(workDir, "config.yaml"), []byte(cfg), 0o644); err != nil { //nolint:gosec // non-secret config, bind-mounted and read by the container
_ = net.Remove(ctx)
return nil, fmt.Errorf("write combined config: %w", err)
}
if err := os.MkdirAll(filepath.Join(workDir, "data"), 0o755); err != nil {
_ = net.Remove(ctx)
return nil, fmt.Errorf("create datadir: %w", err)
}
req := testcontainers.ContainerRequest{
Image: combinedImageTag,
ExposedPorts: []string{combinedHTTPPort},
Image: combinedImage,
ExposedPorts: []string{combinedHTTPPort},
Networks: []string{net.Name},
NetworkAliases: map[string][]string{net.Name: {combinedAlias}},
Env: map[string]string{
"NB_SETUP_PAT_ENABLED": "true",
// Skip the GeoLite DB download — it blocks startup and agent-network
// ingest doesn't use geolocation.
"NB_DISABLE_GEOLOCATION": "true",
},
Cmd: []string{"--config", "/nb/config.yaml"},
HostConfigModifier: func(hc *container.HostConfig) {
@@ -87,7 +116,7 @@ func StartCombined(ctx context.Context) (*Combined, error) {
WaitingFor: wait.ForHTTP("/api/instance").
WithPort(combinedHTTPPort).
WithStatusCodeMatcher(func(status int) bool { return status == 200 }).
WithStartupTimeout(90 * time.Second),
WithStartupTimeout(120 * time.Second),
}
c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
@@ -95,50 +124,109 @@ func StartCombined(ctx context.Context) (*Combined, error) {
Started: true,
})
if err != nil {
_ = net.Remove(ctx)
return nil, fmt.Errorf("start combined container: %w", err)
}
host, err := c.Host(ctx)
if err != nil {
_ = c.Terminate(ctx)
_ = net.Remove(ctx)
return nil, fmt.Errorf("container host: %w", err)
}
mapped, err := c.MappedPort(ctx, nat.Port(combinedHTTPPort))
if err != nil {
_ = c.Terminate(ctx)
_ = net.Remove(ctx)
return nil, fmt.Errorf("mapped port: %w", err)
}
return &Combined{
container: c,
network: net,
BaseURL: fmt.Sprintf("http://%s:%s", host, mapped.Port()),
workDir: workDir,
}, nil
}
// buildCombinedImage builds the combined server image via the docker CLI with
// BuildKit enabled, so the Dockerfile's cache mounts work and unchanged sources
// reuse the layer + go caches. The image is tagged stably so reruns are cheap.
func buildCombinedImage(ctx context.Context, root string) error {
// resolveImage returns the image to run for a component. The env override (or
// the default) is used as-is when it looks like a registry reference (contains
// "/") — testcontainers pulls it. A bare local tag is built from the repo
// Dockerfile instead, so developers can still test source builds.
func resolveImage(ctx context.Context, root, envKey, defaultImage, dockerfile string) (string, error) {
img := defaultImage
if v := os.Getenv(envKey); v != "" {
img = v
}
if strings.Contains(img, "/") {
return img, nil
}
if err := buildImage(ctx, root, dockerfile, img); err != nil {
return "", err
}
return img, nil
}
// buildImage builds an image from a repo Dockerfile via the docker CLI with
// BuildKit enabled, so cache mounts work and unchanged sources reuse the layer
// + go caches. Tags are stable so reruns are cheap.
func buildImage(ctx context.Context, root, dockerfile, tag string) error {
cmd := exec.CommandContext(ctx, "docker", "build",
"-f", combinedDockerfile,
"-t", combinedImageTag,
"-f", dockerfile,
"-t", tag,
".",
)
cmd.Dir = root
cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1")
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("build combined image: %w\n%s", err, string(out))
return fmt.Errorf("build image %s: %w\n%s", tag, err, string(out))
}
return nil
}
// Terminate stops the container and removes the work dir.
// CreateProxyTokenCLI mints a proxy access token via the server's `token
// create` CLI inside the container — the same path the manual install uses.
// This yields a GLOBAL (account-less) token, so the proxy serves the whole
// cluster (SynthesizeServicesForCluster); an account-scoped REST token instead
// drives the per-account path. Returns the plaintext token.
func (c *Combined) CreateProxyTokenCLI(ctx context.Context, name string) (string, error) {
code, reader, err := c.container.Exec(ctx,
[]string{"/go/bin/netbird-server", "token", "create", "--name", name, "--config", "/nb/config.yaml"},
tcexec.Multiplexed())
if err != nil {
return "", fmt.Errorf("exec token create: %w", err)
}
out, _ := io.ReadAll(reader)
if code != 0 {
return "", fmt.Errorf("token create exited %d: %s", code, string(out))
}
for _, line := range strings.Split(string(out), "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "Token:") {
tok := strings.TrimSpace(strings.TrimPrefix(line, "Token:"))
if tok != "" {
return tok, nil
}
}
}
return "", fmt.Errorf("token not found in CLI output: %s", string(out))
}
// Logs returns the combined server container logs, for diagnostics.
func (c *Combined) Logs(ctx context.Context) string {
return containerLogs(ctx, c.container)
}
// Terminate stops the container, removes the shared network, and cleans the
// work dir.
func (c *Combined) Terminate(ctx context.Context) error {
var err error
if c.container != nil {
err = c.container.Terminate(ctx)
}
if c.network != nil {
_ = c.network.Remove(ctx)
}
if c.workDir != "" {
_ = os.RemoveAll(c.workDir)
}

2
e2e/harness/config.go
View File

@@ -17,6 +17,8 @@ const combinedConfigYAML = `server:
logFile: "console"
authSecret: "e2e-relay-secret"
dataDir: "/nb/data"
disableAnonymousMetrics: true
disableGeoliteUpdate: true
auth:
issuer: "%s"
store:

116
e2e/harness/proxy.go Normal file
View File

@@ -0,0 +1,116 @@
//go:build e2e
package harness
import (
"context"
"fmt"
"os"
"time"
"github.com/docker/docker/api/types/container"
"github.com/testcontainers/testcontainers-go"
"github.com/testcontainers/testcontainers-go/wait"
)
const (
proxyDockerfile = "proxy/Dockerfile.multistage"
// defaultProxyImage is the published reverse-proxy release used by default.
// Override with NB_E2E_PROXY_IMAGE; a value without a "/" is built locally.
defaultProxyImage = "netbirdio/reverse-proxy:0.74.0-rc.2"
proxyAlias = "proxy"
// AgentNetworkCluster is the proxy cluster the e2e provider bootstraps and
// the proxy serves. It must equal the management's exposed domain
// (combinedAlias) — the working manual setup uses one NETBIRD_DOMAIN for
// both. The agent-network endpoint is <subdomain>.<cluster>.
AgentNetworkCluster = combinedAlias
)
// Proxy is a running agent-network gateway (netbird proxy) container.
type Proxy struct {
container testcontainers.Container
workDir string
}
// StartProxy builds the proxy image and runs it on the combined server's
// network, registered via the given account proxy token and serving the
// AgentNetworkCluster over a self-signed wildcard cert. It does not wait for
// peer connectivity — callers poll management for the proxy peer.
func StartProxy(ctx context.Context, c *Combined, proxyToken string) (*Proxy, error) {
root, err := repoRoot()
if err != nil {
return nil, err
}
proxyImage, err := resolveImage(ctx, root, "NB_E2E_PROXY_IMAGE", defaultProxyImage, proxyDockerfile)
if err != nil {
return nil, err
}
workDir, err := os.MkdirTemp("/tmp", "nb-e2e-proxy-*")
if err != nil {
return nil, fmt.Errorf("create proxy work dir: %w", err)
}
if err := writeSelfSignedCert(workDir, []string{"*." + AgentNetworkCluster, AgentNetworkCluster}); err != nil {
return nil, err
}
req := testcontainers.ContainerRequest{
Image: proxyImage,
Networks: []string{c.network.Name},
NetworkAliases: map[string][]string{c.network.Name: {proxyAlias}},
Env: map[string]string{
"NB_PROXY_TOKEN": proxyToken,
"NB_PROXY_MANAGEMENT_ADDRESS": combinedExposedURL,
"NB_PROXY_DOMAIN": AgentNetworkCluster,
"NB_PROXY_ADDRESS": ":443",
"NB_PROXY_CERTIFICATE_DIRECTORY": "/certs",
"NB_PROXY_HEALTH_ADDRESS": ":8081",
"NB_PROXY_LOG_LEVEL": "debug",
"NB_PROXY_PRIVATE": "true",
// Management is plain HTTP in-cluster, so allow the proxy token to
// ride a non-TLS gRPC connection.
"NB_PROXY_ALLOW_INSECURE": "true",
// The combined server multiplexes the relay over WebSocket on :8080
// (no QUIC listener). The proxy's embedded relay client defaults to
// QUIC, which fails here and flaps the relay link, churning the
// proxy peer so it never stably registers. Force WS transport.
"NB_RELAY_TRANSPORT": "ws",
// Trace the embedded client (relay / signal / handshake) so
// peer-registration issues are visible in the proxy logs.
"NB_PROXY_CLIENT_LOG_LEVEL": "trace",
},
HostConfigModifier: func(hc *container.HostConfig) {
hc.Binds = append(hc.Binds, workDir+":/certs")
hc.CapAdd = append(hc.CapAdd, "NET_ADMIN", "SYS_ADMIN", "SYS_RESOURCE", "NET_BIND_SERVICE")
},
WaitingFor: wait.ForLog("Initial mapping sync complete").WithStartupTimeout(90 * time.Second),
}
ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
ContainerRequest: req,
Started: true,
})
if err != nil {
return nil, fmt.Errorf("start proxy container: %w", err)
}
return &Proxy{container: ctr, workDir: workDir}, nil
}
// Logs returns the proxy container logs, for diagnostics on failure.
func (p *Proxy) Logs(ctx context.Context) string {
return containerLogs(ctx, p.container)
}
// Terminate stops the proxy container and cleans its work dir.
func (p *Proxy) Terminate(ctx context.Context) error {
var err error
if p.container != nil {
err = p.container.Terminate(ctx)
}
if p.workDir != "" {
_ = os.RemoveAll(p.workDir)
}
return err
}