mirror of
https://github.com/netbirdio/netbird.git
synced 2026-07-23 00:41:29 +02:00
Compare commits
3 Commits
fix/compon
...
vertex-gua
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
16b65729ae | ||
|
|
1b7d23ee9f | ||
|
|
a69fe9ba8c |
@@ -104,8 +104,9 @@ func availableProviders() []providerCase {
|
||||
}
|
||||
|
||||
// providerRequest builds a create request for a matrix provider: enabled, with
|
||||
// a uniquely-priced model for body-routed providers and none for the
|
||||
// path-routed Vertex (whose model lives in the request path).
|
||||
// a uniquely-priced model registered under the id an operator would paste —
|
||||
// the normalized catalog id for Bedrock, the raw form elsewhere (including the
|
||||
// "@version" Vertex id, which the router must normalize to route).
|
||||
func providerRequest(pc providerCase) api.AgentNetworkProviderRequest {
|
||||
req := api.AgentNetworkProviderRequest{
|
||||
Name: pc.name,
|
||||
@@ -114,18 +115,15 @@ func providerRequest(pc providerCase) api.AgentNetworkProviderRequest {
|
||||
ApiKey: &pc.apiKey,
|
||||
Enabled: ptr(true),
|
||||
}
|
||||
if pc.kind != harness.WireVertex {
|
||||
// The router matches the normalized catalog id. Bedrock's request model
|
||||
// travels as a region-prefixed inference-profile id in the URL path
|
||||
// (us.anthropic...), which the router strips before matching, so register
|
||||
// the normalized form here or routing fails as model_not_routable.
|
||||
modelID := pc.model
|
||||
if pc.kind == harness.WireBedrock {
|
||||
modelID = catalogModel(pc)
|
||||
}
|
||||
req.Models = &[]api.AgentNetworkProviderModel{
|
||||
{Id: modelID, InputPer1k: 0.001, OutputPer1k: 0.002},
|
||||
}
|
||||
// Register Bedrock under its normalized catalog id (the router strips the
|
||||
// region prefix before matching); other providers, incl. the raw "@version"
|
||||
// Vertex id, register as-is so the router's normalization is exercised.
|
||||
modelID := pc.model
|
||||
if pc.kind == harness.WireBedrock {
|
||||
modelID = catalogModel(pc)
|
||||
}
|
||||
req.Models = &[]api.AgentNetworkProviderModel{
|
||||
{Id: modelID, InputPer1k: 0.001, OutputPer1k: 0.002},
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
@@ -125,7 +125,13 @@ func TestModelAllowlistEnforced(t *testing.T) {
|
||||
require.NoError(t, perr, "create provider %s", pc.name)
|
||||
id := prov.Id
|
||||
ids = append(ids, id)
|
||||
allowed = append(allowed, catalogModel(pc))
|
||||
// Vertex allowlists the raw "@version" id (the guardrail normalizes it);
|
||||
// other providers allowlist the normalized catalog id.
|
||||
if pc.kind == harness.WireVertex {
|
||||
allowed = append(allowed, pc.model)
|
||||
} else {
|
||||
allowed = append(allowed, catalogModel(pc))
|
||||
}
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), id) })
|
||||
}
|
||||
|
||||
@@ -182,6 +188,22 @@ func TestModelAllowlistEnforced(t *testing.T) {
|
||||
// the upstream), regardless of whether it is a real catalog model.
|
||||
assert.Equal(t, 403, sendModel(ctx, t, cl, settings.Endpoint, proxyIP, pc, disallowedModel(pc)),
|
||||
"model outside the allowlist must be denied for %s", pc.name)
|
||||
|
||||
if pc.kind != harness.WireVertex {
|
||||
return
|
||||
}
|
||||
// Unversioned model id (the customer-reported shape) must pass the
|
||||
// same allowlist entry.
|
||||
assert.Equal(t, 200, sendModel(ctx, t, cl, settings.Endpoint, proxyIP, pc, catalogModel(pc)),
|
||||
"unversioned model id must be permitted for %s", pc.name)
|
||||
// count-tokens carries the real model in the body: allowed → served.
|
||||
code, _, err := cl.VertexCountTokens(ctx, settings.Endpoint, proxyIP, pc.project, pc.region, pc.model)
|
||||
require.NoError(t, err, "count-tokens request must reach the proxy for %s", pc.name)
|
||||
assert.Equal(t, 200, code, "count-tokens with an allowlisted body model must be permitted for %s", pc.name)
|
||||
// Disallowed body model → denied before the upstream.
|
||||
code, _, err = cl.VertexCountTokens(ctx, settings.Endpoint, proxyIP, pc.project, pc.region, disallowedModel(pc))
|
||||
require.NoError(t, err, "count-tokens request must reach the proxy for %s", pc.name)
|
||||
assert.Equal(t, 403, code, "count-tokens with a body model outside the allowlist must be denied for %s", pc.name)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -231,6 +231,15 @@ func (cl *Client) Vertex(ctx context.Context, endpoint, proxyIP, project, region
|
||||
return cl.post(ctx, endpoint, proxyIP, path, body, withSessionID(nil, sessionID))
|
||||
}
|
||||
|
||||
// VertexCountTokens issues the Anthropic-on-Vertex token-count POST: the URL
|
||||
// carries the "count-tokens" pseudo-model and the real model travels in the
|
||||
// body, so the proxy must resolve the body model for the allowlist and routing.
|
||||
func (cl *Client) VertexCountTokens(ctx context.Context, endpoint, proxyIP, project, region, model string) (int, string, error) {
|
||||
path := fmt.Sprintf("/v1/projects/%s/locations/%s/publishers/anthropic/models/count-tokens:rawPredict", project, region)
|
||||
body := fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":"hi"}]}`, model)
|
||||
return cl.post(ctx, endpoint, proxyIP, path, body, nil)
|
||||
}
|
||||
|
||||
// Bedrock issues a native AWS Bedrock InvokeModel POST over the tunnel. The
|
||||
// model id is carried in the request path (/model/{id}/invoke), so the proxy
|
||||
// routes by path; the body uses the bedrock anthropic_version rather than a
|
||||
|
||||
14
proxy/internal/llm/vertex_model.go
Normal file
14
proxy/internal/llm/vertex_model.go
Normal file
@@ -0,0 +1,14 @@
|
||||
package llm
|
||||
|
||||
import "strings"
|
||||
|
||||
// NormalizeVertexModel strips the "@version" suffix from a Vertex AI model id
|
||||
// so it matches the catalog key, e.g. "claude-sonnet-4-5@20250929" ->
|
||||
// "claude-sonnet-4-5". Shared by the parser, router, and guardrail so both
|
||||
// spellings compare equal.
|
||||
func NormalizeVertexModel(modelID string) string {
|
||||
if at := strings.Index(modelID, "@"); at >= 0 {
|
||||
return modelID[:at]
|
||||
}
|
||||
return modelID
|
||||
}
|
||||
20
proxy/internal/llm/vertex_model_test.go
Normal file
20
proxy/internal/llm/vertex_model_test.go
Normal file
@@ -0,0 +1,20 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNormalizeVertexModel(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"claude-sonnet-4-5@20250929": "claude-sonnet-4-5",
|
||||
"claude-opus-4-6@20250514": "claude-opus-4-6",
|
||||
"claude-opus-4-6": "claude-opus-4-6", // bare id passes through
|
||||
"text-embedding-005@001": "text-embedding-005",
|
||||
"@cf/meta/llama-3-8b": "", // leading "@" -> empty (treated as no model)
|
||||
}
|
||||
for in, want := range cases {
|
||||
require.Equal(t, want, NormalizeVertexModel(in), "normalize %q", in)
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"context"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/netbirdio/netbird/proxy/internal/llm"
|
||||
"github.com/netbirdio/netbird/proxy/internal/middleware"
|
||||
)
|
||||
|
||||
@@ -162,6 +163,11 @@ func (m *Middleware) modelInAllowlist(model string) bool {
|
||||
if allowed == normalised {
|
||||
return true
|
||||
}
|
||||
// Accept a Vertex entry stored with its "@version" suffix against the
|
||||
// suffix-stripped request model. Entries without "@" stay exact.
|
||||
if v := llm.NormalizeVertexModel(allowed); v != "" && v != allowed && v == normalised {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -70,6 +70,25 @@ func TestAllowlistMatchAllows(t *testing.T) {
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision, "model in allowlist must be allowed")
|
||||
}
|
||||
|
||||
// A Vertex "@version" allowlist entry must match the version-stripped request
|
||||
// model the parser emits.
|
||||
func TestAllowlistVertexVersionedEntryMatchesStrippedModel(t *testing.T) {
|
||||
mw := New(Config{ModelAllowlist: []string{"claude-opus-4-6@20250514"}})
|
||||
out, err := mw.Invoke(context.Background(), newInput(
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: "claude-opus-4-6"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision,
|
||||
"@version allowlist entry must match the version-stripped request model")
|
||||
|
||||
out, err = mw.Invoke(context.Background(), newInput(
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: "claude-opus-4-8"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, middleware.DecisionDeny, out.Decision,
|
||||
"a different model must stay denied")
|
||||
}
|
||||
|
||||
func TestAllowlistMissDenies(t *testing.T) {
|
||||
mw := New(Config{ModelAllowlist: []string{"gpt-4o"}})
|
||||
out, err := mw.Invoke(context.Background(), newInput(
|
||||
|
||||
@@ -104,3 +104,93 @@ func TestModelAllowlist_URLRoutedProviders(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestModelAllowlist_VertexRequestShapes replays the Vertex request shapes an
|
||||
// Anthropic SDK client sends (model in the URL path, optionally unversioned,
|
||||
// plus the count-tokens body-model endpoint) against bare and "@version"
|
||||
// allowlists. URLs mirror a customer-reported request.
|
||||
func TestModelAllowlist_VertexRequestShapes(t *testing.T) {
|
||||
const (
|
||||
opusBare = "/v1/projects/corp-gcp-it-all-claude/locations/global/publishers/anthropic/models/claude-opus-4-6:rawPredict"
|
||||
opusBareSSE = "/v1/projects/corp-gcp-it-all-claude/locations/global/publishers/anthropic/models/claude-opus-4-6:streamRawPredict"
|
||||
countTokens = "/v1/projects/corp-gcp-it-all-claude/locations/global/publishers/anthropic/models/count-tokens:rawPredict"
|
||||
messagesBody = `{"anthropic_version":"vertex-2023-10-16","messages":[{"role":"user","content":"hi"}]}`
|
||||
countOpusBody = `{"model":"claude-opus-4-6","messages":[{"role":"user","content":"hi"}]}`
|
||||
)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
url string
|
||||
body string
|
||||
allowlist []string
|
||||
decision middleware.Decision
|
||||
denyCode string
|
||||
}{
|
||||
{
|
||||
name: "unversioned model allowed by bare catalog entry",
|
||||
url: opusBare,
|
||||
body: messagesBody,
|
||||
allowlist: []string{"claude-opus-4-6"},
|
||||
decision: middleware.DecisionAllow,
|
||||
},
|
||||
{
|
||||
name: "unversioned model allowed by @version allowlist entry",
|
||||
url: opusBare,
|
||||
body: messagesBody,
|
||||
allowlist: []string{"claude-opus-4-6@20250514"},
|
||||
decision: middleware.DecisionAllow,
|
||||
},
|
||||
{
|
||||
name: "streaming action allowed the same as rawPredict",
|
||||
url: opusBareSSE,
|
||||
body: messagesBody,
|
||||
allowlist: []string{"claude-opus-4-6"},
|
||||
decision: middleware.DecisionAllow,
|
||||
},
|
||||
{
|
||||
// The customer report: a Sonnet-only allowlist must block Opus.
|
||||
name: "unversioned model outside the allowlist denied",
|
||||
url: opusBare,
|
||||
body: messagesBody,
|
||||
allowlist: []string{"claude-sonnet-4-5"},
|
||||
decision: middleware.DecisionDeny,
|
||||
denyCode: "llm_policy.model_blocked",
|
||||
},
|
||||
{
|
||||
name: "count-tokens resolves the body model and passes when allowed",
|
||||
url: countTokens,
|
||||
body: countOpusBody,
|
||||
allowlist: []string{"claude-opus-4-6"},
|
||||
decision: middleware.DecisionAllow,
|
||||
},
|
||||
{
|
||||
name: "count-tokens with a disallowed body model denied",
|
||||
url: countTokens,
|
||||
body: countOpusBody,
|
||||
allowlist: []string{"claude-sonnet-4-5"},
|
||||
decision: middleware.DecisionDeny,
|
||||
denyCode: "llm_policy.model_blocked",
|
||||
},
|
||||
{
|
||||
// No body model: the pseudo-model stays and fails closed.
|
||||
name: "count-tokens without a body model fails closed",
|
||||
url: countTokens,
|
||||
body: messagesBody,
|
||||
allowlist: []string{"claude-opus-4-6"},
|
||||
decision: middleware.DecisionDeny,
|
||||
denyCode: "llm_policy.model_blocked",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
out := runParserGuardrail(t, tt.url, []byte(tt.body), tt.allowlist)
|
||||
assert.Equal(t, tt.decision, out.Decision, "unexpected decision for %s", tt.name)
|
||||
if tt.decision == middleware.DecisionDeny {
|
||||
require.NotNil(t, out.DenyReason, "deny reason must be set for %s", tt.name)
|
||||
assert.Equal(t, 403, out.DenyStatus, "deny status must be 403 for %s", tt.name)
|
||||
assert.Equal(t, tt.denyCode, out.DenyReason.Code, "deny code for %s", tt.name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -253,9 +253,7 @@ func parseVertexPath(reqPath string) (vertexRequest, bool) {
|
||||
if c := strings.LastIndex(rest, ":"); c >= 0 {
|
||||
model, action = rest[:c], rest[c+1:]
|
||||
}
|
||||
if at := strings.Index(model, "@"); at >= 0 {
|
||||
model = model[:at]
|
||||
}
|
||||
model = llm.NormalizeVertexModel(model)
|
||||
if model == "" {
|
||||
return vertexRequest{}, false
|
||||
}
|
||||
@@ -276,24 +274,40 @@ func vertexPublisherVendor(publisher string) string {
|
||||
}
|
||||
}
|
||||
|
||||
// vertexCountTokensModel is the pseudo-model of the Vertex token-count endpoint,
|
||||
// the one Vertex shape whose real model lives in the body, not the URL path.
|
||||
const vertexCountTokensModel = "count-tokens"
|
||||
|
||||
// invokeVertex emits the model/vendor/session/prompt for a Vertex publisher
|
||||
// request, using the publisher's parser to read the (vendor-native) body.
|
||||
func (m middlewareImpl) invokeVertex(in *middleware.Input, vx vertexRequest) *middleware.Output {
|
||||
out := &middleware.Output{Decision: middleware.DecisionAllow}
|
||||
vendor := vertexPublisherVendor(vx.publisher)
|
||||
|
||||
md := []middleware.KV{}
|
||||
if vendor != "" {
|
||||
md = append(md, middleware.KV{Key: middleware.KeyLLMProvider, Value: vendor})
|
||||
}
|
||||
md = append(md, middleware.KV{Key: middleware.KeyLLMModel, Value: vx.model})
|
||||
md = append(md, middleware.KV{Key: middleware.KeyLLMStream, Value: strconv.FormatBool(vx.stream)})
|
||||
|
||||
var parser llm.Parser
|
||||
if vendor != "" {
|
||||
parser, _ = llm.ParserByName(vendor)
|
||||
}
|
||||
|
||||
model := vx.model
|
||||
// count-tokens carries its real model in the body; resolve it so the
|
||||
// guardrail and router evaluate the actual model. An unreadable body keeps
|
||||
// the pseudo-model and fails closed downstream.
|
||||
if model == vertexCountTokensModel && parser != nil {
|
||||
if facts, err := parser.ParseRequest(in.Body); err == nil && facts.Model != "" {
|
||||
if bodyModel := llm.NormalizeVertexModel(facts.Model); bodyModel != "" {
|
||||
model = bodyModel
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
md := []middleware.KV{}
|
||||
if vendor != "" {
|
||||
md = append(md, middleware.KV{Key: middleware.KeyLLMProvider, Value: vendor})
|
||||
}
|
||||
md = append(md, middleware.KV{Key: middleware.KeyLLMModel, Value: model})
|
||||
md = append(md, middleware.KV{Key: middleware.KeyLLMStream, Value: strconv.FormatBool(vx.stream)})
|
||||
|
||||
sessionID := sessionIDFromHeaders(in.Headers)
|
||||
if sessionID == "" && parser != nil {
|
||||
sessionID = parser.ExtractSessionID(in.Body)
|
||||
|
||||
@@ -556,14 +556,16 @@ func routeClaimsModel(route ProviderRoute, model string) bool {
|
||||
if candidate == model {
|
||||
return true
|
||||
}
|
||||
// Bedrock request models reach the router already normalized (the parser
|
||||
// strips the region / inference-profile prefix and version suffix), but
|
||||
// the operator may register the raw inference-profile id (e.g.
|
||||
// "us.anthropic.claude-haiku-4-5"). Normalize the candidate so both sides
|
||||
// compare equal; otherwise a native Bedrock request denies as not-routable.
|
||||
// The request model is already normalized by the parser, but the operator
|
||||
// may register the raw id (Bedrock inference-profile, Vertex "@version").
|
||||
// Normalize the candidate so both spellings match; else it denies as
|
||||
// not-routable.
|
||||
if route.Bedrock && llm.NormalizeBedrockModel(candidate) == model {
|
||||
return true
|
||||
}
|
||||
if route.Vertex && llm.NormalizeVertexModel(candidate) == model {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package llm_router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/proxy/internal/middleware"
|
||||
)
|
||||
|
||||
// A Vertex route registered with the raw "@version" id must match the
|
||||
// version-stripped request model; non-Vertex routes stay exact.
|
||||
func TestRouteClaimsModel_VertexNormalizesCandidate(t *testing.T) {
|
||||
route := ProviderRoute{Vertex: true, Models: []string{"claude-opus-4-6@20250514"}}
|
||||
assert.True(t, routeClaimsModel(route, "claude-opus-4-6"))
|
||||
assert.False(t, routeClaimsModel(route, "claude-haiku-4-5"))
|
||||
|
||||
bare := ProviderRoute{Vertex: true, Models: []string{"claude-opus-4-6"}}
|
||||
assert.True(t, routeClaimsModel(bare, "claude-opus-4-6"))
|
||||
|
||||
direct := ProviderRoute{Models: []string{"claude-opus-4-6@20250514"}}
|
||||
assert.False(t, routeClaimsModel(direct, "claude-opus-4-6"),
|
||||
"non-Vertex routes must not normalize @version candidates")
|
||||
}
|
||||
|
||||
// TestRouter_VertexUnversionedModelRoutes replays the customer-reported request:
|
||||
// an unversioned model id must route on a provider registered with "@version" ids.
|
||||
func TestRouter_VertexUnversionedModelRoutes(t *testing.T) {
|
||||
route := vertexRoute()
|
||||
route.Models = []string{"claude-opus-4-6@20250514", "claude-sonnet-4-5@20250929"}
|
||||
mw := New(Config{Providers: []ProviderRoute{route}})
|
||||
|
||||
in := pathRoutedInput(
|
||||
"/v1/projects/corp-gcp-it-all-claude/locations/global/publishers/anthropic/models/claude-opus-4-6:rawPredict",
|
||||
"anthropic",
|
||||
"claude-opus-4-6",
|
||||
)
|
||||
out, err := mw.Invoke(context.Background(), in)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision,
|
||||
"unversioned request model must route on a provider registered with @version ids")
|
||||
|
||||
denied := pathRoutedInput(
|
||||
"/v1/projects/corp-gcp-it-all-claude/locations/global/publishers/anthropic/models/claude-haiku-4-5:rawPredict",
|
||||
"anthropic",
|
||||
"claude-haiku-4-5",
|
||||
)
|
||||
out, err = mw.Invoke(context.Background(), denied)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, middleware.DecisionDeny, out.Decision,
|
||||
"a model outside the registered list must still deny")
|
||||
require.NotNil(t, out.DenyReason)
|
||||
assert.Equal(t, denyCodeNotRoutable, out.DenyReason.Code)
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
package proxy_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/test/bufconn"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
|
||||
agentNetworkTypes "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
|
||||
rpservice "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service"
|
||||
mgmtgrpc "github.com/netbirdio/netbird/management/internals/shared/grpc"
|
||||
"github.com/netbirdio/netbird/management/server/store"
|
||||
"github.com/netbirdio/netbird/proxy/internal/middleware"
|
||||
"github.com/netbirdio/netbird/proxy/internal/middleware/bodytap"
|
||||
mwbuiltin "github.com/netbirdio/netbird/proxy/internal/middleware/builtin"
|
||||
_ "github.com/netbirdio/netbird/proxy/internal/middleware/builtin/cost_meter"
|
||||
_ "github.com/netbirdio/netbird/proxy/internal/middleware/builtin/llm_guardrail"
|
||||
_ "github.com/netbirdio/netbird/proxy/internal/middleware/builtin/llm_identity_inject"
|
||||
_ "github.com/netbirdio/netbird/proxy/internal/middleware/builtin/llm_limit_check"
|
||||
_ "github.com/netbirdio/netbird/proxy/internal/middleware/builtin/llm_limit_record"
|
||||
_ "github.com/netbirdio/netbird/proxy/internal/middleware/builtin/llm_request_parser"
|
||||
_ "github.com/netbirdio/netbird/proxy/internal/middleware/builtin/llm_response_parser"
|
||||
_ "github.com/netbirdio/netbird/proxy/internal/middleware/builtin/llm_router"
|
||||
"github.com/netbirdio/netbird/proxy/internal/proxy"
|
||||
nbproxytypes "github.com/netbirdio/netbird/proxy/internal/types"
|
||||
"github.com/netbirdio/netbird/shared/management/proto"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// TestReverseProxy_VertexGuardrail_ModelAllowlist drives Anthropic-on-Vertex
|
||||
// requests (model in the URL path, not the body) through the full synthesized
|
||||
// middleware chain against an in-process management stack and a fake upstream.
|
||||
// With a Sonnet-only guardrail, Opus must be denied (model_blocked) before the
|
||||
// upstream and Sonnet must reach it. Two provider shapes exercise different
|
||||
// code: "catch_all" (no models → only the guardrail can block Opus) and
|
||||
// "versioned_models" (raw "@version" ids → the router must normalize to route
|
||||
// Sonnet while the guardrail still blocks Opus).
|
||||
func TestReverseProxy_VertexGuardrail_ModelAllowlist(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("sqlite store not supported on Windows")
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
providerModels []agentNetworkTypes.ProviderModel
|
||||
}{
|
||||
{name: "catch_all", providerModels: nil},
|
||||
{name: "versioned_models", providerModels: []agentNetworkTypes.ProviderModel{
|
||||
{ID: "claude-sonnet-4-5@20250929"},
|
||||
{ID: "claude-opus-4-6@20250514"},
|
||||
}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
tc := tc
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
runVertexGuardrailCase(t, tc.providerModels)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func runVertexGuardrailCase(t *testing.T, providerModels []agentNetworkTypes.ProviderModel) {
|
||||
t.Helper()
|
||||
|
||||
const (
|
||||
testAccountID = "acct-vertex-guard-1"
|
||||
testAdminUser = "user-admin-1"
|
||||
adminGroupID = "grp-admins"
|
||||
providerID = "prov-vertex-test"
|
||||
guardrailID = "ainguard-sonnet-only"
|
||||
cluster = "test.proxy.local"
|
||||
subdomain = "vertexguard"
|
||||
)
|
||||
testLogger := log.New()
|
||||
testLogger.SetLevel(log.PanicLevel)
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Fake Vertex upstream: a hit on the disallowed model means a guardrail miss.
|
||||
var upstreamHits atomic.Int64
|
||||
upstreamBody := []byte(`{"id":"msg_x","type":"message","role":"assistant","model":"claude","content":[{"type":"text","text":"pong"}],"usage":{"input_tokens":5,"output_tokens":2}}`)
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
upstreamHits.Add(1)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write(upstreamBody)
|
||||
}))
|
||||
t.Cleanup(upstream.Close)
|
||||
|
||||
// In-process management gRPC (bufconn) over a real sqlite store + manager.
|
||||
st, cleanup, err := store.NewTestStoreFromSQL(ctx, "", t.TempDir())
|
||||
require.NoError(t, err, "real sqlite test store must come up")
|
||||
t.Cleanup(cleanup)
|
||||
|
||||
anMgr := agentnetwork.NewManager(st, nil, nil, nil)
|
||||
server := &mgmtgrpc.ProxyServiceServer{}
|
||||
server.SetAgentNetworkLimitsService(anMgr)
|
||||
|
||||
lis := bufconn.Listen(1024 * 1024)
|
||||
srv := grpc.NewServer()
|
||||
proto.RegisterProxyServiceServer(srv, server)
|
||||
go func() { _ = srv.Serve(lis) }()
|
||||
t.Cleanup(srv.Stop)
|
||||
|
||||
conn, err := grpc.NewClient("passthrough:///bufnet",
|
||||
grpc.WithContextDialer(func(_ context.Context, _ string) (net.Conn, error) { return lis.Dial() }),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
mgmtClient := proto.NewProxyServiceClient(conn)
|
||||
|
||||
require.NoError(t, st.SaveAgentNetworkSettings(ctx, &agentNetworkTypes.Settings{
|
||||
AccountID: testAccountID,
|
||||
Cluster: cluster,
|
||||
Subdomain: subdomain,
|
||||
}))
|
||||
require.NoError(t, st.SaveAgentNetworkProvider(ctx, &agentNetworkTypes.Provider{
|
||||
ID: providerID,
|
||||
AccountID: testAccountID,
|
||||
ProviderID: "vertex_ai_api",
|
||||
Name: "vertex-guard-test",
|
||||
UpstreamURL: upstream.URL,
|
||||
// A static bearer (not "keyfile::…") so the router injects a static auth
|
||||
// header instead of minting a GCP token, which needs network egress and
|
||||
// would deny before the guardrail runs, masking the decision under test.
|
||||
APIKey: "static-vertex-token",
|
||||
Enabled: true,
|
||||
Models: providerModels,
|
||||
SessionPrivateKey: "priv",
|
||||
SessionPublicKey: "pub",
|
||||
}))
|
||||
// Guardrail allowlisting ONLY Sonnet.
|
||||
require.NoError(t, st.SaveAgentNetworkGuardrail(ctx, &agentNetworkTypes.Guardrail{
|
||||
ID: guardrailID,
|
||||
AccountID: testAccountID,
|
||||
Name: "sonnet-only",
|
||||
Checks: agentNetworkTypes.GuardrailChecks{
|
||||
ModelAllowlist: agentNetworkTypes.GuardrailModelAllowlist{
|
||||
Enabled: true,
|
||||
Models: []string{"claude-sonnet-4-5"},
|
||||
},
|
||||
},
|
||||
}))
|
||||
require.NoError(t, st.SaveAgentNetworkPolicy(ctx, &agentNetworkTypes.Policy{
|
||||
ID: "ainpol-vertex-guard",
|
||||
AccountID: testAccountID,
|
||||
Name: "admins-vertex",
|
||||
Enabled: true,
|
||||
SourceGroups: []string{adminGroupID},
|
||||
DestinationProviderIDs: []string{providerID},
|
||||
GuardrailIDs: []string{guardrailID},
|
||||
}))
|
||||
|
||||
services, err := agentnetwork.SynthesizeServices(ctx, st, testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, services, 1, "exactly one synth service expected")
|
||||
synthSvc := services[0]
|
||||
require.NotEmpty(t, synthSvc.Targets, "synth target must exist")
|
||||
|
||||
mwbuiltin.Configure(ctx, t.TempDir(), nil, testLogger, mgmtClient)
|
||||
registry := mwbuiltin.DefaultRegistry()
|
||||
mwMetrics, err := middleware.NewMetrics(nil)
|
||||
require.NoError(t, err)
|
||||
mwMgr := middleware.NewManager(0, mwMetrics, testLogger)
|
||||
mwMgr.SetResolver(middleware.NewResolver(registry))
|
||||
|
||||
specs := make([]middleware.Spec, 0, len(synthSvc.Targets[0].Options.Middlewares))
|
||||
for _, mw := range synthSvc.Targets[0].Options.Middlewares {
|
||||
var slot middleware.Slot
|
||||
switch mw.Slot {
|
||||
case rpservice.MiddlewareSlotOnRequest:
|
||||
slot = middleware.SlotOnRequest
|
||||
case rpservice.MiddlewareSlotOnResponse:
|
||||
slot = middleware.SlotOnResponse
|
||||
case rpservice.MiddlewareSlotTerminal:
|
||||
slot = middleware.SlotTerminal
|
||||
default:
|
||||
t.Fatalf("unknown middleware slot %q on %s", mw.Slot, mw.ID)
|
||||
}
|
||||
specs = append(specs, middleware.Spec{
|
||||
ID: mw.ID,
|
||||
Slot: slot,
|
||||
Enabled: mw.Enabled,
|
||||
FailMode: middleware.FailOpen,
|
||||
Timeout: middleware.DefaultTimeout,
|
||||
RawConfig: append([]byte(nil), mw.ConfigJSON...),
|
||||
CanMutate: mw.CanMutate,
|
||||
})
|
||||
}
|
||||
|
||||
serviceIDStr := synthSvc.ID
|
||||
require.NoError(t, mwMgr.Rebuild(serviceIDStr, []middleware.PathTargetBinding{{
|
||||
ServiceID: serviceIDStr,
|
||||
PathID: "/",
|
||||
Specs: specs,
|
||||
}}))
|
||||
|
||||
upstreamURL, err := url.Parse(upstream.URL)
|
||||
require.NoError(t, err)
|
||||
|
||||
rp := proxy.NewReverseProxy(http.DefaultTransport, "auto", nil, testLogger, proxy.WithMiddlewareManager(mwMgr))
|
||||
rp.AddMapping(proxy.Mapping{
|
||||
ID: nbproxytypes.ServiceID(serviceIDStr),
|
||||
AccountID: nbproxytypes.AccountID(testAccountID),
|
||||
Host: synthSvc.Domain,
|
||||
Paths: map[string]*proxy.PathTarget{
|
||||
"/": {
|
||||
URL: upstreamURL,
|
||||
DirectUpstream: true,
|
||||
AgentNetwork: true,
|
||||
Middlewares: specs,
|
||||
CaptureConfig: &bodytap.Config{
|
||||
MaxRequestBytes: 1 << 20,
|
||||
MaxResponseBytes: 1 << 20,
|
||||
ContentTypes: []string{"application/json", "text/event-stream"},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
// No "model" field — the model lives in the URL path, as Vertex clients send.
|
||||
const vertexBody = `{"anthropic_version":"vertex-2023-10-16","max_tokens":64,"messages":[{"role":"user","content":"Reply with exactly: pong"}]}`
|
||||
|
||||
send := func(t *testing.T, model string) (int, int64, string) {
|
||||
t.Helper()
|
||||
before := upstreamHits.Load()
|
||||
path := "/v1/projects/corp-gcp-it-all-claude/locations/global/publishers/anthropic/models/" + model + ":rawPredict"
|
||||
req := httptest.NewRequest("POST", "https://"+synthSvc.Domain+path, strings.NewReader(vertexBody))
|
||||
req.Host = synthSvc.Domain
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
cd := proxy.NewCapturedData("req-" + model)
|
||||
cd.SetServiceID(nbproxytypes.ServiceID(serviceIDStr))
|
||||
cd.SetAccountID(nbproxytypes.AccountID(testAccountID))
|
||||
cd.SetUserID(testAdminUser)
|
||||
cd.SetUserGroups([]string{adminGroupID})
|
||||
cd.SetAuthMethod("tunnel_peer")
|
||||
req = req.WithContext(proxy.WithCapturedData(req.Context(), cd))
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
rp.ServeHTTP(w, req)
|
||||
return w.Code, upstreamHits.Load() - before, w.Body.String()
|
||||
}
|
||||
|
||||
// Opus must be denied by the guardrail (model_blocked, not the router's
|
||||
// model_not_routable) before reaching the upstream — the customer-reported bug.
|
||||
t.Run("opus_denied_by_guardrail", func(t *testing.T) {
|
||||
code, hits, body := send(t, "claude-opus-4-6")
|
||||
assert.Equal(t, http.StatusForbidden, code, "Opus must be denied under a Sonnet-only allowlist; body=%s", body)
|
||||
assert.Contains(t, body, "llm_policy.model_blocked", "denial must come from the guardrail allowlist, not routing; body=%s", body)
|
||||
assert.Equal(t, int64(0), hits, "a denied request must never reach the Vertex upstream")
|
||||
})
|
||||
|
||||
// The allowed model (Sonnet) passes the guardrail and reaches the upstream.
|
||||
t.Run("sonnet_allowed", func(t *testing.T) {
|
||||
code, hits, body := send(t, "claude-sonnet-4-5")
|
||||
assert.Equal(t, http.StatusOK, code, "Sonnet is allowlisted and must be served; body=%s", body)
|
||||
assert.Equal(t, int64(1), hits, "the allowed request must reach the Vertex upstream exactly once")
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user