//go:build e2e package harness import ( "bufio" "context" "encoding/json" "fmt" "io" "net" "net/http" "net/url" "os" "path/filepath" "strings" "sync/atomic" "time" "github.com/docker/docker/api/types/container" "github.com/docker/go-connections/nat" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" ) const ( httpUpstreamImage = "nginx:alpine" httpUpstreamAlias = "httpupstream" httpUpstreamPort = "8080/tcp" httpUpstreamLogPath = "/tmp/e2e-access.log" ) // httpUpstreamNginxConf returns request details from a plain HTTP upstream and // writes every request URI to an unbuffered file. Tests use unique path markers // to distinguish requests that reached the upstream from proxy-generated // responses with the same status. const httpUpstreamNginxConf = `pid /tmp/nginx.pid; worker_processes 1; events {} http { log_format e2e escape=json '{"uri":"$request_uri"}'; access_log /tmp/e2e-access.log e2e; server { listen 8080; location / { default_type application/json; return 200 '{"source":"netbird-e2e-http-upstream","path":"$uri","request_uri":"$request_uri","configured_auth":"$http_x_e2e_proxy_key","authorization":"$http_authorization","cookie":"$http_cookie","netbird_user":"$http_x_netbird_user","netbird_groups":"$http_x_netbird_groups"}'; } } } ` // HTTPUpstream is a plain HTTP server on the combined server's network. It // reflects selected request properties and records request URIs for tests that // need to distinguish proxy responses from upstream responses. type HTTPUpstream struct { container testcontainers.Container workDir string directAddress string syncSequence atomic.Uint64 URL string } // StartHTTPUpstream runs a request-observing nginx server on the shared network. func StartHTTPUpstream(ctx context.Context, c *Combined) (*HTTPUpstream, error) { workDir, err := os.MkdirTemp("/tmp", "nb-e2e-http-upstream-*") if err != nil { return nil, fmt.Errorf("create HTTP upstream work dir: %w", err) } if err := os.Chmod(workDir, 0o755); err != nil { //nolint:gosec // throwaway e2e config dir, must be traversable by the container uid _ = os.RemoveAll(workDir) return nil, fmt.Errorf("chmod HTTP upstream dir: %w", err) } if err := os.WriteFile(filepath.Join(workDir, "nginx.conf"), []byte(httpUpstreamNginxConf), 0o644); err != nil { //nolint:gosec // non-secret e2e config _ = os.RemoveAll(workDir) return nil, fmt.Errorf("write HTTP upstream config: %w", err) } req := testcontainers.ContainerRequest{ Image: httpUpstreamImage, ExposedPorts: []string{httpUpstreamPort}, Networks: []string{c.network.Name}, NetworkAliases: map[string][]string{c.network.Name: {httpUpstreamAlias}}, Cmd: []string{"nginx", "-c", "/conf/nginx.conf", "-g", "daemon off;"}, HostConfigModifier: func(hc *container.HostConfig) { hc.Binds = append(hc.Binds, workDir+":/conf:ro") }, WaitingFor: wait.ForListeningPort(httpUpstreamPort).WithStartupTimeout(60 * time.Second), } ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: req, Started: true, }) if err != nil { cleanupHTTPUpstreamStart(ctr, workDir) return nil, fmt.Errorf("start HTTP upstream container: %w", err) } host, err := ctr.Host(ctx) if err != nil { cleanupHTTPUpstreamStart(ctr, workDir) return nil, fmt.Errorf("HTTP upstream container host: %w", err) } mapped, err := ctr.MappedPort(ctx, nat.Port(httpUpstreamPort)) if err != nil { cleanupHTTPUpstreamStart(ctr, workDir) return nil, fmt.Errorf("HTTP upstream mapped port: %w", err) } return &HTTPUpstream{ container: ctr, workDir: workDir, directAddress: net.JoinHostPort(host, mapped.Port()), URL: "http://" + httpUpstreamAlias + ":8080", }, nil } func cleanupHTTPUpstreamStart(ctr testcontainers.Container, workDir string) { if ctr != nil { cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) _ = ctr.Terminate(cleanupCtx) cancel() } _ = os.RemoveAll(workDir) } // Synchronize waits until the upstream has finalized all requests received // before a direct barrier request. The fixture runs one nginx worker and writes // access logs without buffering, so observing the barrier in the file proves // any earlier proxied request has also been recorded. func (u *HTTPUpstream) Synchronize(ctx context.Context) error { marker := fmt.Sprintf("__e2e_upstream_barrier_%d__", u.syncSequence.Add(1)) requestURL := (&url.URL{ Scheme: "http", Host: u.directAddress, Path: "/" + marker, }).String() req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) if err != nil { return fmt.Errorf("create HTTP upstream barrier request: %w", err) } transport := &http.Transport{} defer transport.CloseIdleConnections() client := &http.Client{Transport: transport, Timeout: 5 * time.Second} resp, err := client.Do(req) if err != nil { return fmt.Errorf("send HTTP upstream barrier request: %w", err) } defer resp.Body.Close() if _, err := io.Copy(io.Discard, resp.Body); err != nil { return fmt.Errorf("read HTTP upstream barrier response: %w", err) } if resp.StatusCode != http.StatusOK { return fmt.Errorf("HTTP upstream barrier returned status %d", resp.StatusCode) } waitCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() ticker := time.NewTicker(25 * time.Millisecond) defer ticker.Stop() for { count, err := u.RequestCount(waitCtx, marker) if err != nil { return err } switch { case count == 1: return nil case count > 1: return fmt.Errorf("HTTP upstream recorded barrier %q %d times", marker, count) } select { case <-waitCtx.Done(): return fmt.Errorf("wait for HTTP upstream barrier: %w", waitCtx.Err()) case <-ticker.C: } } } // RequestCount returns the number of recorded request URIs containing marker. func (u *HTTPUpstream) RequestCount(ctx context.Context, marker string) (int, error) { if marker == "" { return 0, fmt.Errorf("request marker is empty") } reader, err := u.container.CopyFileFromContainer(ctx, httpUpstreamLogPath) if err != nil { return 0, fmt.Errorf("read HTTP upstream access log: %w", err) } defer reader.Close() count := 0 scanner := bufio.NewScanner(reader) for scanner.Scan() { var entry struct { URI string `json:"uri"` } if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil { return 0, fmt.Errorf("decode HTTP upstream access log: %w", err) } if strings.Contains(entry.URI, marker) { count++ } } if err := scanner.Err(); err != nil { return 0, fmt.Errorf("scan HTTP upstream access log: %w", err) } return count, nil } // Logs returns the HTTP upstream container logs for failure diagnostics. func (u *HTTPUpstream) Logs(ctx context.Context) string { return containerLogs(ctx, u.container) } // Terminate stops the HTTP upstream container and removes its work directory. func (u *HTTPUpstream) Terminate(ctx context.Context) error { var err error if u.container != nil { err = u.container.Terminate(ctx) } if u.workDir != "" { _ = os.RemoveAll(u.workDir) } return err }