netbird/e2e/harness/combined.go

//go:build e2e

package harness

import (
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/docker/docker/api/types/container"
	"github.com/docker/go-connections/nat"
	"github.com/testcontainers/testcontainers-go"
	tcexec "github.com/testcontainers/testcontainers-go/exec"
	"github.com/testcontainers/testcontainers-go/network"
	"github.com/testcontainers/testcontainers-go/wait"

	"github.com/netbirdio/netbird/shared/management/client/rest"
)

const (
	combinedDockerfile = "combined/Dockerfile.multistage"
	// defaultCombinedImage is the published combined-server release used by
	// default (the artifact operators run manually). Override with
	// NB_E2E_COMBINED_IMAGE; a value without a "/" is built locally from
	// combinedDockerfile instead of pulled.
	defaultCombinedImage = "netbirdio/netbird-server:0.74.0-rc.2"
	combinedHTTPPort     = "8080/tcp"

	// combinedAlias is the combined server's network alias AND the deployment
	// domain. The working manual setup uses a single NETBIRD_DOMAIN for the
	// management exposed address, the proxy domain, and the agent-network
	// cluster — so we mirror that: peers reach management/signal/relay at this
	// name, the proxy registers this as its cluster, and the agent-network
	// endpoint is <subdomain>.<combinedAlias>.
	combinedAlias      = "netbird.local"
	combinedExposedURL = "http://" + combinedAlias + ":8080"

	// containerIssuer is the embedded IdP issuer, used only for internal JWT
	// validation (peers authenticate with setup keys / proxy tokens, not OIDC),
	// so the in-container localhost address is fine.
	containerIssuer = "http://localhost:8080/oauth2"
)

// Combined is a running combined NetBird server (management + signal + relay +
// STUN + embedded IdP) plus the connection details tests need. It owns the
// shared docker network that the proxy and client containers join.
type Combined struct {
	container testcontainers.Container
	network   *testcontainers.DockerNetwork
	// BaseURL is the host-reachable management API root, e.g. http://127.0.0.1:51234.
	BaseURL string
	// PAT is the admin Personal Access Token minted via Bootstrap.
	PAT string

	api     *rest.Client
	workDir string
}

// StartCombined builds the combined server from its multistage Dockerfile and
// boots it with setup-PAT enabled on a fresh shared network, returning once the
// API is serving. The caller still owns minting the admin PAT via Bootstrap.
func StartCombined(ctx context.Context) (*Combined, error) {
	root, err := repoRoot()
	if err != nil {
		return nil, err
	}

	combinedImage, err := resolveImage(ctx, root, "NB_E2E_COMBINED_IMAGE", defaultCombinedImage, combinedDockerfile)
	if err != nil {
		return nil, err
	}

	net, err := network.New(ctx)
	if err != nil {
		return nil, fmt.Errorf("create shared network: %w", err)
	}

	// Work dir under /tmp so Docker Desktop file sharing (which excludes
	// macOS's /var/folders TMPDIR) can bind-mount it.
	workDir, err := os.MkdirTemp("/tmp", "nb-e2e-combined-*")
	if err != nil {
		_ = net.Remove(ctx)
		return nil, fmt.Errorf("create work dir: %w", err)
	}

	cfg := fmt.Sprintf(combinedConfigYAML, combinedExposedURL, containerIssuer)
	if err := os.WriteFile(filepath.Join(workDir, "config.yaml"), []byte(cfg), 0o644); err != nil { //nolint:gosec // non-secret config, bind-mounted and read by the container
		_ = net.Remove(ctx)
		return nil, fmt.Errorf("write combined config: %w", err)
	}
	if err := os.MkdirAll(filepath.Join(workDir, "data"), 0o755); err != nil {
		_ = net.Remove(ctx)
		return nil, fmt.Errorf("create datadir: %w", err)
	}

	req := testcontainers.ContainerRequest{
		Image:          combinedImage,
		ExposedPorts:   []string{combinedHTTPPort},
		Networks:       []string{net.Name},
		NetworkAliases: map[string][]string{net.Name: {combinedAlias}},
		Env: map[string]string{
			"NB_SETUP_PAT_ENABLED": "true",
			// Skip the GeoLite DB download — it blocks startup and agent-network
			// ingest doesn't use geolocation.
			"NB_DISABLE_GEOLOCATION": "true",
		},
		Cmd: []string{"--config", "/nb/config.yaml"},
		HostConfigModifier: func(hc *container.HostConfig) {
			hc.Binds = append(hc.Binds, workDir+":/nb")
		},
		WaitingFor: wait.ForHTTP("/api/instance").
			WithPort(combinedHTTPPort).
			WithStatusCodeMatcher(func(status int) bool { return status == 200 }).
			WithStartupTimeout(120 * time.Second),
	}

	c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
		ContainerRequest: req,
		Started:          true,
	})
	if err != nil {
		_ = net.Remove(ctx)
		return nil, fmt.Errorf("start combined container: %w", err)
	}

	host, err := c.Host(ctx)
	if err != nil {
		_ = c.Terminate(ctx)
		_ = net.Remove(ctx)
		return nil, fmt.Errorf("container host: %w", err)
	}
	mapped, err := c.MappedPort(ctx, nat.Port(combinedHTTPPort))
	if err != nil {
		_ = c.Terminate(ctx)
		_ = net.Remove(ctx)
		return nil, fmt.Errorf("mapped port: %w", err)
	}

	return &Combined{
		container: c,
		network:   net,
		BaseURL:   fmt.Sprintf("http://%s:%s", host, mapped.Port()),
		workDir:   workDir,
	}, nil
}

// resolveImage returns the image to run for a component. The env override (or
// the default) is used as-is when it looks like a registry reference (contains
// "/") — testcontainers pulls it. A bare local tag is built from the repo
// Dockerfile instead, so developers can still test source builds.
func resolveImage(ctx context.Context, root, envKey, defaultImage, dockerfile string) (string, error) {
	img := defaultImage
	if v := os.Getenv(envKey); v != "" {
		img = v
	}
	if strings.Contains(img, "/") {
		return img, nil
	}
	if err := buildImage(ctx, root, dockerfile, img); err != nil {
		return "", err
	}
	return img, nil
}

// buildImage builds an image from a repo Dockerfile via the docker CLI with
// BuildKit enabled, so cache mounts work and unchanged sources reuse the layer
// + go caches. Tags are stable so reruns are cheap.
func buildImage(ctx context.Context, root, dockerfile, tag string) error {
	cmd := exec.CommandContext(ctx, "docker", "build",
		"-f", dockerfile,
		"-t", tag,
		".",
	)
	cmd.Dir = root
	cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1")
	if out, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("build image %s: %w\n%s", tag, err, string(out))
	}
	return nil
}

// CreateProxyTokenCLI mints a proxy access token via the server's `token
// create` CLI inside the container — the same path the manual install uses.
// This yields a GLOBAL (account-less) token, so the proxy serves the whole
// cluster (SynthesizeServicesForCluster); an account-scoped REST token instead
// drives the per-account path. Returns the plaintext token.
func (c *Combined) CreateProxyTokenCLI(ctx context.Context, name string) (string, error) {
	code, reader, err := c.container.Exec(ctx,
		[]string{"/go/bin/netbird-server", "token", "create", "--name", name, "--config", "/nb/config.yaml"},
		tcexec.Multiplexed())
	if err != nil {
		return "", fmt.Errorf("exec token create: %w", err)
	}
	out, _ := io.ReadAll(reader)
	if code != 0 {
		return "", fmt.Errorf("token create exited %d: %s", code, string(out))
	}
	for _, line := range strings.Split(string(out), "\n") {
		line = strings.TrimSpace(line)
		if strings.HasPrefix(line, "Token:") {
			tok := strings.TrimSpace(strings.TrimPrefix(line, "Token:"))
			if tok != "" {
				return tok, nil
			}
		}
	}
	return "", fmt.Errorf("token not found in CLI output: %s", string(out))
}

// Logs returns the combined server container logs, for diagnostics.
func (c *Combined) Logs(ctx context.Context) string {
	return containerLogs(ctx, c.container)
}

// Terminate stops the container, removes the shared network, and cleans the
// work dir.
func (c *Combined) Terminate(ctx context.Context) error {
	var err error
	if c.container != nil {
		err = c.container.Terminate(ctx)
	}
	if c.network != nil {
		_ = c.network.Remove(ctx)
	}
	if c.workDir != "" {
		_ = os.RemoveAll(c.workDir)
	}
	return err
}