[management,proxy] Add per-provider skip_tls_verification for agent-network (#6630)

* [management,proxy] Add per-provider skip_tls_verification for agent-network Let agent-network providers opt into skipping upstream TLS verification for self-hosted / internal gateways behind a private or self-signed cert. - provider: add SkipTLSVerification (persisted via AutoMigrate) with request/response mapping (nil on update preserves, explicit false clears). - openapi: skip_tls_verification on the provider request + response; types regenerated. - synthesizer: carry the flag into the llm_router route config so it reaches the proxy. - proxy: llm_router sets it on the UpstreamRewrite mutation, and the reverse proxy applies roundtrip.WithSkipTLSVerify per selected route when forwarding upstream (the router dials per provider, so a per-target flag alone wouldn't cover it). - tests: synthesizer route config carries the flag, router rewrite propagates it, and the request/response round-trip incl. update semantics. * [e2e] Validate per-provider skip_tls_verification end to end Add a self-signed HTTPS upstream (nginx) to the harness and a test that provisions two providers on that same upstream — one with skip_tls_verification=true, one false — behind one proxy + client. The skip=true provider's chat reaches the upstream (200); the skip=false provider's fails the TLS handshake (5xx). Same upstream, opposite outcome, which proves the flag is honoured per provider (a single target-level flag could not, since all of an account's providers share one synthesised target). * [e2e] WaitProxyPeer: require >=1 connected peer, not exact 1/1 Each proxy container registers a fresh WireGuard key and its peer is not removed on teardown, so proxy peers from earlier tests linger in the account as disconnected. WaitProxyPeer matched the exact string "1/1 Connected", which failed once a second proxy-using test ran in the same package (status "1/2"). Parse the "Peers count: X/Y Connected" line and wait for X>=1 instead: only the live proxy can be connected, and the caller's subsequent chat is the real end-to-end assertion. Fixes the CI failure of TestProviderSkipTLSVerification (runs after TestProvidersMatrix).
2026-09-19 05:09:06 +02:00 · 2026-07-01 20:43:15 +02:00
parent 0aa0f7c76b
commit eb422a5cd3
14 changed files with 456 additions and 12 deletions
@@ -0,0 +1,140 @@
+//go:build e2e
+
+package agentnetwork
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/e2e/harness"
+	"github.com/netbirdio/netbird/shared/management/http/api"
+)
+
+// TestProviderSkipTLSVerification proves skip_tls_verification is per-provider:
+// two providers share one self-signed upstream, one skipping TLS verification
+// and one not. The skip=true provider's chat reaches the upstream and returns
+// 200; the skip=false provider's chat fails at the TLS handshake — same
+// upstream, opposite outcome. This is the behaviour a target-level flag could
+// not give, since all of an account's providers share one synthesised target.
+func TestProviderSkipTLSVerification(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	defer cancel()
+
+	up, err := harness.StartFakeUpstream(ctx, srv)
+	require.NoError(t, err, "start self-signed upstream")
+	t.Cleanup(func() { _ = up.Terminate(context.Background()) })
+
+	grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-skiptls"})
+	require.NoError(t, err, "create group")
+	t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
+
+	ephemeral := false
+	sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
+		Name:       "e2e-skiptls-client",
+		Type:       "reusable",
+		ExpiresIn:  86400,
+		UsageLimit: 0,
+		AutoGroups: []string{grp.Id},
+		Ephemeral:  &ephemeral,
+	})
+	require.NoError(t, err, "mint setup key")
+	require.NotEmpty(t, sk.Key, "setup key plaintext")
+
+	const (
+		insecureModel = "insecure-model"
+		secureModel   = "secure-model"
+	)
+
+	// Two providers on the SAME self-signed upstream, distinguished only by their
+	// skip_tls_verification and a unique model string so the router picks each
+	// unambiguously.
+	newReq := func(name, model string, skip bool) api.AgentNetworkProviderRequest {
+		key := "sk-dummy-e2e"
+		return api.AgentNetworkProviderRequest{
+			Name:                name,
+			ProviderId:          "openai_api",
+			UpstreamUrl:         up.URL,
+			ApiKey:              &key,
+			Enabled:             ptr(true),
+			SkipTlsVerification: ptr(skip),
+			Models: &[]api.AgentNetworkProviderModel{
+				{Id: model, InputPer1k: 0.001, OutputPer1k: 0.002},
+			},
+		}
+	}
+
+	// First create bootstraps the account cluster.
+	insecureReq := newReq("skip-tls", insecureModel, true)
+	insecureReq.BootstrapCluster = ptr(harness.AgentNetworkCluster)
+	insecureProv, err := srv.CreateProvider(ctx, insecureReq)
+	require.NoError(t, err, "create skip-tls provider")
+	t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), insecureProv.Id) })
+	require.True(t, insecureProv.SkipTlsVerification, "response must echo skip_tls_verification=true")
+
+	secureProv, err := srv.CreateProvider(ctx, newReq("verify-tls", secureModel, false))
+	require.NoError(t, err, "create verify-tls provider")
+	t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), secureProv.Id) })
+	require.False(t, secureProv.SkipTlsVerification, "response must echo skip_tls_verification=false")
+
+	enabled := true
+	pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
+		Name:                   "e2e-skiptls-allow",
+		Enabled:                &enabled,
+		SourceGroups:           []string{grp.Id},
+		DestinationProviderIds: []string{insecureProv.Id, secureProv.Id},
+	})
+	require.NoError(t, err, "create policy")
+	t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
+
+	settings, err := srv.GetSettings(ctx)
+	require.NoError(t, err, "read settings")
+	require.NotEmpty(t, settings.Endpoint, "endpoint must be assigned")
+
+	proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-skiptls-proxy")
+	require.NoError(t, err, "mint proxy token")
+	px, err := harness.StartProxy(ctx, srv, proxyToken)
+	require.NoError(t, err, "start proxy")
+	t.Cleanup(func() { _ = px.Terminate(context.Background()) })
+
+	cl, err := harness.StartClient(ctx, srv, sk.Key)
+	require.NoError(t, err, "start client")
+	t.Cleanup(func() { _ = cl.Terminate(context.Background()) })
+
+	require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management")
+	if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil {
+		t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background()))
+	}
+	proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint)
+	require.NoError(t, err, "resolve endpoint to proxy IP")
+
+	// Positive: skip=true reaches the self-signed upstream. Retry to absorb
+	// tunnel/DNS jitter on the first call; success also proves the path works.
+	var code int
+	var body string
+	deadline := time.Now().Add(90 * time.Second)
+	for time.Now().Before(deadline) {
+		c, b, cerr := cl.Chat(ctx, settings.Endpoint, proxyIP, harness.WireChat, insecureModel, "Reply with exactly: pong", "e2e-skiptls-insecure")
+		if cerr == nil {
+			code, body = c, b
+			if code == 200 {
+				break
+			}
+		}
+		time.Sleep(5 * time.Second)
+	}
+	require.Equal(t, 200, code,
+		"skip_tls_verification=true must reach the self-signed upstream; body: %s\n=== upstream logs ===\n%s\n=== proxy logs ===\n%s",
+		body, up.Logs(context.Background()), px.Logs(context.Background()))
+
+	// Negative: skip=false must fail the TLS handshake to the SAME upstream. The
+	// path is already proven working, so a non-200 here is the cert rejection.
+	secureCode, secureBody, cerr := cl.Chat(ctx, settings.Endpoint, proxyIP, harness.WireChat, secureModel, "Reply with exactly: pong", "e2e-skiptls-secure")
+	require.NoError(t, cerr, "the chat call itself must complete (proxy returns an error status, not a transport error)")
+	require.NotEqual(t, 200, secureCode,
+		"skip_tls_verification=false must NOT reach the self-signed upstream; got %d, body: %s", secureCode, secureBody)
+	require.GreaterOrEqual(t, secureCode, 500,
+		"a TLS verification failure should surface as a 5xx from the proxy; got %d, body: %s", secureCode, secureBody)
+}
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"os/exec"
+	"strconv"
 	"strings"
 	"time"

@@ -108,9 +109,48 @@ func (cl *Client) WaitConnected(ctx context.Context, timeout time.Duration) erro
 	return cl.pollStatus(ctx, timeout, "Management: Connected")
 }

-// WaitProxyPeer polls until the client sees the proxy peer connected (1/1).
+// WaitProxyPeer polls until the client sees at least one connected peer — the
+// proxy serving the agent-network endpoint. It requires ">=1 connected" rather
+// than an exact "1/1" because proxy peers from earlier tests linger in the
+// account as disconnected (each proxy container registers a fresh WireGuard key
+// and the peer is not removed on teardown), so the count is e.g. "1/2". Only the
+// live proxy can be connected, and the caller's subsequent chat is the real
+// end-to-end assertion.
 func (cl *Client) WaitProxyPeer(ctx context.Context, timeout time.Duration) error {
-	return cl.pollStatus(ctx, timeout, "1/1 Connected")
+	deadline := time.Now().Add(timeout)
+	var last string
+	for time.Now().Before(deadline) {
+		out, _ := cl.Status(ctx)
+		last = out
+		if connectedPeers(out) >= 1 {
+			return nil
+		}
+		time.Sleep(3 * time.Second)
+	}
+	return fmt.Errorf("timed out waiting for a connected proxy peer; last status:\n%s", last)
+}
+
+// connectedPeers parses the "Peers count: X/Y Connected" line from `netbird
+// status` and returns X (the connected count), or 0 when absent/unparseable.
+func connectedPeers(status string) int {
+	for _, line := range strings.Split(status, "\n") {
+		line = strings.TrimSpace(line)
+		rest, ok := strings.CutPrefix(line, "Peers count:")
+		if !ok {
+			continue
+		}
+		rest = strings.TrimSpace(rest)
+		slash := strings.IndexByte(rest, '/')
+		if slash <= 0 {
+			return 0
+		}
+		n, err := strconv.Atoi(strings.TrimSpace(rest[:slash]))
+		if err != nil {
+			return 0
+		}
+		return n
+	}
+	return 0
 }

 func (cl *Client) pollStatus(ctx context.Context, timeout time.Duration, want string) error {
@@ -0,0 +1,107 @@
+//go:build e2e
+
+package harness
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/wait"
+)
+
+const (
+	fakeUpstreamImage = "nginx:alpine"
+	fakeUpstreamAlias = "fakeupstream"
+	fakeUpstreamPort  = "443/tcp"
+)
+
+// fakeUpstreamNginxConf serves a canned OpenAI-shaped chat completion for any
+// path over a self-signed certificate, so the proxy reaches it only when the
+// provider opts into skipping TLS verification.
+const fakeUpstreamNginxConf = `pid /tmp/nginx.pid;
+events {}
+http {
+  server {
+    listen 443 ssl;
+    ssl_certificate     /certs/tls.crt;
+    ssl_certificate_key /certs/tls.key;
+    location / {
+      default_type application/json;
+      return 200 '{"id":"chatcmpl-e2e","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":"pong"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}';
+    }
+  }
+}
+`
+
+// FakeUpstream is a self-signed HTTPS server on the combined server's network,
+// used to exercise provider skip_tls_verification: a proxy that verifies the
+// certificate rejects it, one that skips verification reaches it.
+type FakeUpstream struct {
+	container testcontainers.Container
+	workDir   string
+	// URL is the upstream URL providers point at (https://<alias>).
+	URL string
+}
+
+// StartFakeUpstream runs the self-signed upstream on the shared network.
+func StartFakeUpstream(ctx context.Context, c *Combined) (*FakeUpstream, error) {
+	workDir, err := os.MkdirTemp("/tmp", "nb-e2e-upstream-*")
+	if err != nil {
+		return nil, fmt.Errorf("create upstream work dir: %w", err)
+	}
+	// Widen so the (non-root worker) nginx container can traverse the bind mount.
+	if err := os.Chmod(workDir, 0o755); err != nil { //nolint:gosec // throwaway e2e cert dir
+		return nil, fmt.Errorf("chmod upstream dir: %w", err)
+	}
+	if err := writeSelfSignedCert(workDir, []string{fakeUpstreamAlias}); err != nil {
+		return nil, err
+	}
+	if err := os.WriteFile(filepath.Join(workDir, "nginx.conf"), []byte(fakeUpstreamNginxConf), 0o644); err != nil { //nolint:gosec // non-secret e2e config
+		return nil, fmt.Errorf("write nginx conf: %w", err)
+	}
+
+	req := testcontainers.ContainerRequest{
+		Image:          fakeUpstreamImage,
+		ExposedPorts:   []string{fakeUpstreamPort},
+		Networks:       []string{c.network.Name},
+		NetworkAliases: map[string][]string{c.network.Name: {fakeUpstreamAlias}},
+		Cmd:            []string{"nginx", "-c", "/certs/nginx.conf", "-g", "daemon off;"},
+		HostConfigModifier: func(hc *container.HostConfig) {
+			hc.Binds = append(hc.Binds, workDir+":/certs:ro")
+		},
+		WaitingFor: wait.ForListeningPort(fakeUpstreamPort).WithStartupTimeout(60 * time.Second),
+	}
+
+	ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: req,
+		Started:          true,
+	})
+	if err != nil {
+		_ = os.RemoveAll(workDir)
+		return nil, fmt.Errorf("start fake upstream container: %w", err)
+	}
+
+	return &FakeUpstream{container: ctr, workDir: workDir, URL: "https://" + fakeUpstreamAlias}, nil
+}
+
+// Logs returns the upstream container logs, for diagnostics on failure.
+func (u *FakeUpstream) Logs(ctx context.Context) string {
+	return containerLogs(ctx, u.container)
+}
+
+// Terminate stops the upstream container and cleans its work dir.
+func (u *FakeUpstream) Terminate(ctx context.Context) error {
+	var err error
+	if u.container != nil {
+		err = u.container.Terminate(ctx)
+	}
+	if u.workDir != "" {
+		_ = os.RemoveAll(u.workDir)
+	}
+	return err
+}