mirror of
https://github.com/netbirdio/netbird.git
synced 2026-07-29 20:01:29 +02:00
Compare commits
16 Commits
feat/agent
...
mdm_integr
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f91f9fc05c | ||
|
|
e92aa7dfb0 | ||
|
|
e1ffb165a4 | ||
|
|
2f268c8141 | ||
|
|
e3c4128164 | ||
|
|
bab5572a74 | ||
|
|
9b4a5df925 | ||
|
|
1816a020c4 | ||
|
|
aa13928b76 | ||
|
|
d681670a9d | ||
|
|
7715c382ee | ||
|
|
b2c5732847 | ||
|
|
0340893854 | ||
|
|
874195440c | ||
|
|
bec26d5a14 | ||
|
|
db2c9b6f49 |
@@ -273,8 +273,8 @@ dockers_v2:
|
||||
- netbirdio/netbird
|
||||
- ghcr.io/netbirdio/netbird
|
||||
tags:
|
||||
- "v{{ .Version }}-rootless"
|
||||
- "{{ if eq .Env.SKIP_PUBLISH \"false\" }}latest{{ end }}"
|
||||
- "{{ .Version }}-rootless"
|
||||
- "{{ if eq .Env.SKIP_PUBLISH \"false\" }}rootless-latest{{ end }}"
|
||||
dockerfile: client/Dockerfile-rootless
|
||||
extra_files:
|
||||
- client/netbird-entrypoint.sh
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/netbirdio/netbird/client/internal/profilemanager"
|
||||
"github.com/netbirdio/netbird/client/internal/routemanager"
|
||||
"github.com/netbirdio/netbird/client/internal/stdnet"
|
||||
"github.com/netbirdio/netbird/client/mdm"
|
||||
"github.com/netbirdio/netbird/client/net"
|
||||
"github.com/netbirdio/netbird/client/system"
|
||||
"github.com/netbirdio/netbird/formatter"
|
||||
@@ -75,6 +76,13 @@ type Client struct {
|
||||
connectClient *internal.ConnectClient
|
||||
config *profilemanager.Config
|
||||
cacheDir string
|
||||
|
||||
// mdmLoader holds the per-Client MDM policy source. Set by
|
||||
// SetMDMPolicyFetcher (called from the Kotlin side). Each Run
|
||||
// passes this loader to the resolved Config so applyMDMPolicy
|
||||
// picks up the active overlay. Nil means "MDM enforcement off
|
||||
// for this Client".
|
||||
mdmLoader *mdm.Loader
|
||||
}
|
||||
|
||||
func (c *Client) setState(cfg *profilemanager.Config, cacheDir string, cc *internal.ConnectClient) {
|
||||
@@ -129,6 +137,7 @@ func (c *Client) Run(platformFiles PlatformFiles, urlOpener URLOpener, isAndroid
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.applyMDMOverlay(cfg)
|
||||
c.recorder.UpdateManagementAddress(cfg.ManagementURL.String())
|
||||
c.recorder.UpdateRosenpass(cfg.RosenpassEnabled, cfg.RosenpassPermissive)
|
||||
|
||||
@@ -173,6 +182,7 @@ func (c *Client) RunWithoutLogin(platformFiles PlatformFiles, dns *DNSList, dnsR
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.applyMDMOverlay(cfg)
|
||||
c.recorder.UpdateManagementAddress(cfg.ManagementURL.String())
|
||||
c.recorder.UpdateRosenpass(cfg.RosenpassEnabled, cfg.RosenpassPermissive)
|
||||
|
||||
@@ -230,6 +240,7 @@ func (c *Client) DebugBundle(platformFiles PlatformFiles, anonymize bool) (strin
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
c.applyMDMOverlay(cfg)
|
||||
cacheDir = platformFiles.CacheDir()
|
||||
}
|
||||
|
||||
|
||||
80
client/android/mdm.go
Normal file
80
client/android/mdm.go
Normal file
@@ -0,0 +1,80 @@
|
||||
//go:build android
|
||||
|
||||
package android
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/netbirdio/netbird/client/internal/profilemanager"
|
||||
"github.com/netbirdio/netbird/client/mdm"
|
||||
)
|
||||
|
||||
// PolicyFetcher is the mobile-side bridge for the MDM managed-config
|
||||
// snapshot. The native layer (Kotlin) implements this and registers
|
||||
// the instance per Client via Client.SetMDMPolicyFetcher. Every
|
||||
// invocation of fetchJSON must read the current RestrictionsManager
|
||||
// state and return the result as a JSON-encoded map[string]any string.
|
||||
//
|
||||
// JSON is used because gomobile does not support map[string]any
|
||||
// crossing the JNI boundary — the adapter on the Go side parses the
|
||||
// string back into the map[string]any expected by mdm.Loader.
|
||||
//
|
||||
// Return value contract:
|
||||
// - "" (empty) : interpreted as "no MDM source / no managed keys"
|
||||
// - "{}" : managed config explicitly empty
|
||||
// - "{...}" : JSON object with key/value pairs
|
||||
// - malformed JSON : logged and treated as empty
|
||||
type PolicyFetcher interface {
|
||||
FetchJSON() string
|
||||
}
|
||||
|
||||
// jsonFetcherAdapter wraps a gomobile-exposed PolicyFetcher into the
|
||||
// internal mdm.PolicyFetcher interface, taking care of JSON decoding
|
||||
// on every Fetch.
|
||||
type jsonFetcherAdapter struct {
|
||||
inner PolicyFetcher
|
||||
}
|
||||
|
||||
func (a *jsonFetcherAdapter) Fetch() map[string]any {
|
||||
raw := a.inner.FetchJSON()
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal([]byte(raw), &out); err != nil {
|
||||
log.Warnf("MDM mobile fetcher: invalid JSON payload from native: %v", err)
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// SetMDMPolicyFetcher registers the native-provided MDM policy fetcher
|
||||
// on this Client. Call once from the gomobile-init code (Kotlin
|
||||
// Application.onCreate or Service onCreate) before invoking Run /
|
||||
// RunWithoutLogin. Passing nil disables MDM enforcement on this
|
||||
// Client.
|
||||
//
|
||||
// The fetcher is held as a *mdm.Loader instance on the Client (no
|
||||
// package-level state) — multiple Clients in the same process get
|
||||
// independent Loaders, and tests can inject fakes per Client.
|
||||
func (c *Client) SetMDMPolicyFetcher(p PolicyFetcher) {
|
||||
if p == nil {
|
||||
c.mdmLoader = mdm.NewLoader(nil)
|
||||
return
|
||||
}
|
||||
c.mdmLoader = mdm.NewLoader(&jsonFetcherAdapter{inner: p})
|
||||
}
|
||||
|
||||
// applyMDMOverlay applies the Client-held MDM Loader's current policy
|
||||
// on top of the just-read Config. Called immediately after every
|
||||
// UpdateOrCreateConfig — profilemanager's apply() initialises the
|
||||
// policy to empty and leaves overlay responsibility to the lifecycle
|
||||
// owner. No-op when no fetcher was registered.
|
||||
func (c *Client) applyMDMOverlay(cfg *profilemanager.Config) {
|
||||
if cfg == nil || c.mdmLoader == nil {
|
||||
return
|
||||
}
|
||||
cfg.ApplyMDMPolicy(c.mdmLoader.Load())
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/netbirdio/netbird/client/internal"
|
||||
"github.com/netbirdio/netbird/client/internal/auth"
|
||||
"github.com/netbirdio/netbird/client/internal/profilemanager"
|
||||
"github.com/netbirdio/netbird/client/mdm"
|
||||
nbnet "github.com/netbirdio/netbird/client/net"
|
||||
"github.com/netbirdio/netbird/client/proto"
|
||||
"github.com/netbirdio/netbird/client/server"
|
||||
@@ -332,6 +333,11 @@ func doForegroundLogin(ctx context.Context, cmd *cobra.Command, setupKey string,
|
||||
if err != nil {
|
||||
return fmt.Errorf("read config file %s: %v", configFilePath, err)
|
||||
}
|
||||
// CLI standalone login: profilemanager no longer auto-applies MDM,
|
||||
// so layer in the OS-native policy here. Desktop builds construct
|
||||
// a Loader with no fetcher — the build-tagged loadPlatform reads
|
||||
// the registry/plist directly.
|
||||
config.ApplyMDMPolicy(mdm.NewLoader(nil).Load())
|
||||
|
||||
// Mirror runInForegroundMode: recover residual state (DNS, firewall,
|
||||
// ssh config, legacy routing) from a previous unclean shutdown and
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/netbirdio/netbird/client/internal"
|
||||
"github.com/netbirdio/netbird/client/internal/peer"
|
||||
"github.com/netbirdio/netbird/client/internal/profilemanager"
|
||||
"github.com/netbirdio/netbird/client/mdm"
|
||||
"github.com/netbirdio/netbird/client/proto"
|
||||
nbnet "github.com/netbirdio/netbird/client/net"
|
||||
"github.com/netbirdio/netbird/client/server"
|
||||
@@ -228,6 +229,10 @@ func runInForegroundMode(ctx context.Context, cmd *cobra.Command, activeProf *pr
|
||||
if err != nil {
|
||||
return fmt.Errorf("get config file: %v", err)
|
||||
}
|
||||
// CLI foreground path runs without the daemon Server: layer in the
|
||||
// active MDM policy explicitly so a forced ManagementURL / PSK /
|
||||
// other managed key actually takes effect on this run.
|
||||
config.ApplyMDMPolicy(mdm.NewLoader(nil).Load())
|
||||
|
||||
_, _ = profilemanager.UpdateOldManagementURL(ctx, config, configFilePath)
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/netbirdio/netbird/client/internal/auth"
|
||||
"github.com/netbirdio/netbird/client/internal/peer"
|
||||
"github.com/netbirdio/netbird/client/internal/profilemanager"
|
||||
"github.com/netbirdio/netbird/client/mdm"
|
||||
sshcommon "github.com/netbirdio/netbird/client/ssh"
|
||||
"github.com/netbirdio/netbird/client/system"
|
||||
"github.com/netbirdio/netbird/shared/management/domain"
|
||||
@@ -215,6 +216,10 @@ func New(opts Options) (*Client, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create config: %w", err)
|
||||
}
|
||||
// Embedded path runs without the daemon Server: apply the active
|
||||
// MDM policy explicitly so a forced ManagementURL / PSK / other
|
||||
// managed key takes effect on this embedded engine instance.
|
||||
config.ApplyMDMPolicy(mdm.NewLoader(nil).Load())
|
||||
|
||||
if opts.PrivateKey != "" {
|
||||
config.PrivateKey = opts.PrivateKey
|
||||
|
||||
@@ -58,10 +58,6 @@ var DefaultInterfaceBlacklist = []string{
|
||||
"Tailscale", "tailscale", "docker", "veth", "br-", "lo",
|
||||
}
|
||||
|
||||
// loadMDMPolicy is the package-level indirection used by apply() to read the
|
||||
// active MDM policy. Tests override this to inject a fake policy.
|
||||
var loadMDMPolicy = mdm.LoadPolicy
|
||||
|
||||
// ConfigInput carries configuration changes to the client
|
||||
type ConfigInput struct {
|
||||
ManagementURL string
|
||||
@@ -186,14 +182,27 @@ type Config struct {
|
||||
|
||||
MTU uint16
|
||||
|
||||
// policy is the MDM policy that produced the currently-set values for
|
||||
// any MDM-enforced fields. Set by applyMDMPolicy at the tail of apply()
|
||||
// and reset on every apply() invocation. Never persisted to disk.
|
||||
// Callers query enforcement state via Policy() and the mdm.Policy API
|
||||
// (HasKey, ManagedKeys, IsEmpty).
|
||||
// policy is the MDM policy that produced the currently-set values
|
||||
// for any MDM-enforced fields. Set by ApplyMDMPolicy on every
|
||||
// invocation. Never persisted to disk. Callers query enforcement
|
||||
// state via Policy() and the mdm.Policy API (HasKey, ManagedKeys,
|
||||
// IsEmpty).
|
||||
policy *mdm.Policy `json:"-"`
|
||||
}
|
||||
|
||||
// ApplyMDMPolicy overlays the supplied MDM Policy on top of the
|
||||
// currently resolved Config values. Idempotent — pass an empty Policy
|
||||
// to clear any prior overlay. The lifecycle owner (Server.getConfig
|
||||
// on desktop, the Client.Run path on mobile) calls this with
|
||||
// loader.Load() once the per-process Loader is known; the Config
|
||||
// itself holds no reference to the Loader.
|
||||
func (config *Config) ApplyMDMPolicy(policy *mdm.Policy) {
|
||||
if config == nil {
|
||||
return
|
||||
}
|
||||
config.applyMDMPolicy(policy)
|
||||
}
|
||||
|
||||
// Policy returns the MDM policy applied to this Config. Returns a non-nil
|
||||
// empty Policy when MDM enforcement is inactive; callers can always invoke
|
||||
// HasKey / ManagedKeys / IsEmpty without a nil check.
|
||||
@@ -650,9 +659,11 @@ func (config *Config) apply(input ConfigInput) (updated bool, err error) {
|
||||
updated = true
|
||||
}
|
||||
|
||||
// MDM is the last override layer: any key present in the policy
|
||||
// supersedes defaults, on-disk config, env vars and CLI input.
|
||||
config.applyMDMPolicy(loadMDMPolicy())
|
||||
// Initialise the MDM overlay to "no enforcement" so Config.Policy()
|
||||
// never returns a stale or nil policy on a freshly applied Config.
|
||||
// Lifecycle owners that want to enforce a real MDM policy invoke
|
||||
// Config.ApplyMDMPolicy(loader.Load()) after this returns.
|
||||
config.applyMDMPolicy(mdm.NewPolicy(nil))
|
||||
|
||||
return updated, nil
|
||||
}
|
||||
|
||||
@@ -10,24 +10,58 @@ import (
|
||||
"github.com/netbirdio/netbird/client/mdm"
|
||||
)
|
||||
|
||||
// withMDMPolicy temporarily overrides the package-level loadMDMPolicy hook so
|
||||
// apply() observes the supplied Policy. The original loader is restored at
|
||||
// test cleanup.
|
||||
func withMDMPolicy(t *testing.T, policy *mdm.Policy) {
|
||||
// fakeFetcher implements mdm.PolicyFetcher returning a pre-set policy
|
||||
// map. Test helper used to construct a Loader without touching the OS
|
||||
// or any package-level state.
|
||||
type fakeFetcher struct{ values map[string]any }
|
||||
|
||||
func (f *fakeFetcher) Fetch() map[string]any { return f.values }
|
||||
|
||||
// loaderFor builds an mdm.Loader whose loadPlatform returns the
|
||||
// supplied Policy's underlying values.
|
||||
func loaderFor(policy *mdm.Policy) *mdm.Loader {
|
||||
if policy == nil || policy.IsEmpty() {
|
||||
return mdm.NewLoader(&fakeFetcher{values: nil})
|
||||
}
|
||||
values := make(map[string]any)
|
||||
for _, k := range policy.ManagedKeys() {
|
||||
if v, ok := policy.GetString(k); ok {
|
||||
values[k] = v
|
||||
continue
|
||||
}
|
||||
if v, ok := policy.GetBool(k); ok {
|
||||
values[k] = v
|
||||
continue
|
||||
}
|
||||
if v, ok := policy.GetInt(k); ok {
|
||||
values[k] = v
|
||||
continue
|
||||
}
|
||||
if v, ok := policy.GetStringSlice(k); ok {
|
||||
values[k] = v
|
||||
}
|
||||
}
|
||||
return mdm.NewLoader(&fakeFetcher{values: values})
|
||||
}
|
||||
|
||||
// configWithMDM is the test convenience that builds a Config via
|
||||
// UpdateOrCreateConfig and overlays the supplied MDM policy on top —
|
||||
// mirrors the production pattern (Server.getConfig / Client.applyMDMOverlay)
|
||||
// where the Loader lives outside Config and the apply step is driven
|
||||
// by the lifecycle owner.
|
||||
func configWithMDM(t *testing.T, input ConfigInput, policy *mdm.Policy) *Config {
|
||||
t.Helper()
|
||||
prev := loadMDMPolicy
|
||||
loadMDMPolicy = func() *mdm.Policy { return policy }
|
||||
t.Cleanup(func() { loadMDMPolicy = prev })
|
||||
cfg, err := UpdateOrCreateConfig(input)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
cfg.ApplyMDMPolicy(loaderFor(policy).Load())
|
||||
return cfg
|
||||
}
|
||||
|
||||
func TestApply_MDMEmpty_NoEnforcement(t *testing.T) {
|
||||
withMDMPolicy(t, mdm.NewPolicy(nil))
|
||||
|
||||
cfg, err := UpdateOrCreateConfig(ConfigInput{
|
||||
cfg := configWithMDM(t, ConfigInput{
|
||||
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
}, mdm.NewPolicy(nil))
|
||||
|
||||
assert.True(t, cfg.Policy().IsEmpty(), "no MDM source ⇒ empty Policy")
|
||||
assert.False(t, cfg.Policy().HasKey(mdm.KeyManagementURL))
|
||||
@@ -39,18 +73,15 @@ func TestApply_MDMEmpty_NoEnforcement(t *testing.T) {
|
||||
|
||||
func TestApply_MDMOnly_OverridesDefaults(t *testing.T) {
|
||||
const mdmURL = "https://corp.mdm.example.com:443"
|
||||
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
|
||||
|
||||
cfg := configWithMDM(t, ConfigInput{
|
||||
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
|
||||
}, mdm.NewPolicy(map[string]any{
|
||||
mdm.KeyManagementURL: mdmURL,
|
||||
mdm.KeyDisableClientRoutes: true,
|
||||
mdm.KeyBlockInbound: true,
|
||||
}))
|
||||
|
||||
cfg, err := UpdateOrCreateConfig(ConfigInput{
|
||||
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
|
||||
assert.Equal(t, mdmURL, cfg.ManagementURL.String())
|
||||
assert.True(t, cfg.DisableClientRoutes)
|
||||
assert.True(t, cfg.BlockInbound)
|
||||
@@ -65,16 +96,12 @@ func TestApply_MDMBeatsCLIInput(t *testing.T) {
|
||||
const mdmURL = "https://mdm.example.com:443"
|
||||
const cliURL = "https://cli.example.com:443"
|
||||
|
||||
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
|
||||
mdm.KeyManagementURL: mdmURL,
|
||||
}))
|
||||
|
||||
cfg, err := UpdateOrCreateConfig(ConfigInput{
|
||||
cfg := configWithMDM(t, ConfigInput{
|
||||
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
|
||||
ManagementURL: cliURL,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
}, mdm.NewPolicy(map[string]any{
|
||||
mdm.KeyManagementURL: mdmURL,
|
||||
}))
|
||||
|
||||
// MDM wins over CLI-supplied management URL.
|
||||
assert.Equal(t, mdmURL, cfg.ManagementURL.String())
|
||||
@@ -82,16 +109,12 @@ func TestApply_MDMBeatsCLIInput(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestApply_MDMInvalidURL_KeepsPreviousValue(t *testing.T) {
|
||||
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
|
||||
cfg := configWithMDM(t, ConfigInput{
|
||||
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
|
||||
}, mdm.NewPolicy(map[string]any{
|
||||
mdm.KeyManagementURL: "not-a-url",
|
||||
}))
|
||||
|
||||
cfg, err := UpdateOrCreateConfig(ConfigInput{
|
||||
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
|
||||
// Invalid MDM URL is logged and skipped: default URL stays in place
|
||||
// to keep the client functional.
|
||||
assert.Equal(t, DefaultManagementURL, cfg.ManagementURL.String())
|
||||
@@ -106,24 +129,20 @@ func TestApply_MDMBoolKeysOverrideOnDiskValue(t *testing.T) {
|
||||
tmp := filepath.Join(t.TempDir(), "config.json")
|
||||
|
||||
// Seed without MDM.
|
||||
withMDMPolicy(t, mdm.NewPolicy(nil))
|
||||
_, err := UpdateOrCreateConfig(ConfigInput{
|
||||
configWithMDM(t, ConfigInput{
|
||||
ConfigPath: tmp,
|
||||
DisableClientRoutes: boolPtr(false),
|
||||
RosenpassEnabled: boolPtr(false),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}, mdm.NewPolicy(nil))
|
||||
|
||||
// Now enable MDM enforcement for these keys.
|
||||
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
|
||||
cfg := configWithMDM(t, ConfigInput{
|
||||
ConfigPath: tmp,
|
||||
}, mdm.NewPolicy(map[string]any{
|
||||
mdm.KeyDisableClientRoutes: true,
|
||||
mdm.KeyRosenpassEnabled: true,
|
||||
}))
|
||||
|
||||
cfg, err := UpdateOrCreateConfig(ConfigInput{ConfigPath: tmp})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
|
||||
assert.True(t, cfg.DisableClientRoutes, "MDM override should flip on-disk false to true")
|
||||
assert.True(t, cfg.RosenpassEnabled)
|
||||
assert.True(t, cfg.Policy().HasKey(mdm.KeyDisableClientRoutes))
|
||||
@@ -164,16 +183,12 @@ func TestApply_MDMLazyConnection(t *testing.T) {
|
||||
func TestApply_MDMPreSharedKeyRedactionSentinelRejected(t *testing.T) {
|
||||
const maskSentinel = "**********"
|
||||
|
||||
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
|
||||
cfg := configWithMDM(t, ConfigInput{
|
||||
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
|
||||
}, mdm.NewPolicy(map[string]any{
|
||||
mdm.KeyPreSharedKey: maskSentinel,
|
||||
}))
|
||||
|
||||
cfg, err := UpdateOrCreateConfig(ConfigInput{
|
||||
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cfg)
|
||||
|
||||
// Mask sentinel must not be persisted as the actual PSK.
|
||||
assert.NotEqual(t, maskSentinel, cfg.PreSharedKey)
|
||||
// Key still marked managed so user writes are still rejected.
|
||||
|
||||
@@ -95,7 +95,7 @@ func (d *DnsInterceptor) RemoveRoute() error {
|
||||
|
||||
// AllowedIPs should use real IPs
|
||||
if d.currentPeerKey != "" {
|
||||
if _, err := d.allowedIPsRefcounter.Decrement(prefix); err != nil {
|
||||
if _, err := d.allowedIPsRefcounter.Decrement(prefix, d.currentPeerKey); err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %v", prefix, err))
|
||||
}
|
||||
}
|
||||
@@ -172,7 +172,7 @@ func (d *DnsInterceptor) removeAllowedIP(realPrefix netip.Prefix) error {
|
||||
}
|
||||
|
||||
// AllowedIPs use real IPs
|
||||
if _, err := d.allowedIPsRefcounter.Decrement(realPrefix); err != nil {
|
||||
if _, err := d.allowedIPsRefcounter.Decrement(realPrefix, d.currentPeerKey); err != nil {
|
||||
return fmt.Errorf("remove allowed IP %s: %v", realPrefix, err)
|
||||
}
|
||||
|
||||
@@ -205,7 +205,7 @@ func (d *DnsInterceptor) RemoveAllowedIPs() error {
|
||||
for _, prefixes := range d.interceptedDomains {
|
||||
for _, prefix := range prefixes {
|
||||
// AllowedIPs use real IPs
|
||||
if _, err := d.allowedIPsRefcounter.Decrement(prefix); err != nil {
|
||||
if _, err := d.allowedIPsRefcounter.Decrement(prefix, d.currentPeerKey); err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %v", prefix, err))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,7 +135,7 @@ func (r *Route) RemoveAllowedIPs() error {
|
||||
var merr *multierror.Error
|
||||
for _, domainPrefixes := range r.dynamicDomains {
|
||||
for _, prefix := range domainPrefixes {
|
||||
if _, err := r.allowedIPsRefcounter.Decrement(prefix); err != nil {
|
||||
if _, err := r.allowedIPsRefcounter.Decrement(prefix, r.currentPeerKey); err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %w", prefix, err))
|
||||
}
|
||||
}
|
||||
@@ -320,7 +320,7 @@ func (r *Route) removeRoutes(prefixes []netip.Prefix) ([]netip.Prefix, error) {
|
||||
merr = multierror.Append(merr, fmt.Errorf("remove dynamic route for IP %s: %w", prefix, err))
|
||||
}
|
||||
if r.currentPeerKey != "" {
|
||||
if _, err := r.allowedIPsRefcounter.Decrement(prefix); err != nil {
|
||||
if _, err := r.allowedIPsRefcounter.Decrement(prefix, r.currentPeerKey); err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %w", prefix, err))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -216,7 +216,7 @@ func (m *DefaultManager) setupRefCounters(useNoop bool) {
|
||||
)
|
||||
}
|
||||
|
||||
m.allowedIPsRefCounter = refcounter.New(
|
||||
m.allowedIPsRefCounter = refcounter.NewAllowedIPs(
|
||||
func(prefix netip.Prefix, peerKey string) (string, error) {
|
||||
// save peerKey to use it in the remove function
|
||||
return peerKey, m.wgInterface.AddAllowedIP(peerKey, prefix)
|
||||
|
||||
185
client/internal/routemanager/refcounter/allowedips.go
Normal file
185
client/internal/routemanager/refcounter/allowedips.go
Normal file
@@ -0,0 +1,185 @@
|
||||
package refcounter
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
|
||||
nberrors "github.com/netbirdio/netbird/client/errors"
|
||||
)
|
||||
|
||||
// allowedIPsEntry holds the per-peer reference counts for a single prefix and which peer is
|
||||
// currently installed in WireGuard. WireGuard allows a prefix on exactly one peer, so at most
|
||||
// one peer is active at a time even when several peers reference the prefix.
|
||||
type allowedIPsEntry struct {
|
||||
// peers maps a peerKey to the number of references holding the prefix for that peer.
|
||||
peers map[string]int
|
||||
// active is the peerKey currently installed in WireGuard for this prefix ("" if none).
|
||||
active string
|
||||
// total is the sum of all per-peer reference counts (kept in sync with peers).
|
||||
total int
|
||||
}
|
||||
|
||||
// AllowedIPsRefCounter is a peer-aware reference counter for WireGuard AllowedIPs.
|
||||
//
|
||||
// The generic Counter keys only by prefix and remembers a single Out value set by the first
|
||||
// caller, which it never changes. That is wrong for AllowedIPs: two independent watchers (or
|
||||
// multiple resolved domains) can reference the same prefix through different peers, and when the
|
||||
// peer currently installed in WireGuard releases its last reference the prefix must be handed over
|
||||
// to a surviving peer instead of being left pointing at the released one.
|
||||
//
|
||||
// It calls add/remove (which program WireGuard) only on the transitions that matter:
|
||||
// - add on the first reference for a prefix, or when swapping the active peer;
|
||||
// - remove on the last reference for a prefix, or on the old peer during a swap.
|
||||
type AllowedIPsRefCounter struct {
|
||||
mu sync.Mutex
|
||||
entries map[netip.Prefix]*allowedIPsEntry
|
||||
add AddFunc[netip.Prefix, string, string]
|
||||
remove RemoveFunc[netip.Prefix, string]
|
||||
}
|
||||
|
||||
// NewAllowedIPs creates a new peer-aware AllowedIPs reference counter.
|
||||
// add programs a prefix on a peer in WireGuard and returns the peerKey to store as the active peer.
|
||||
// remove unprograms the prefix from the given peer.
|
||||
func NewAllowedIPs(add AddFunc[netip.Prefix, string, string], remove RemoveFunc[netip.Prefix, string]) *AllowedIPsRefCounter {
|
||||
return &AllowedIPsRefCounter{
|
||||
entries: map[netip.Prefix]*allowedIPsEntry{},
|
||||
add: add,
|
||||
remove: remove,
|
||||
}
|
||||
}
|
||||
|
||||
// Increment adds a reference to prefix for peerKey. WireGuard is programmed only for the first
|
||||
// reference to a prefix; while a different peer is already installed the prefix is left with it
|
||||
// (first peer wins, HA at the WireGuard layer is not possible) and only the reference count is kept.
|
||||
func (rm *AllowedIPsRefCounter) Increment(prefix netip.Prefix, peerKey string) (Ref[string], error) {
|
||||
rm.mu.Lock()
|
||||
defer rm.mu.Unlock()
|
||||
|
||||
e, ok := rm.entries[prefix]
|
||||
if !ok {
|
||||
e = &allowedIPsEntry{peers: map[string]int{}}
|
||||
rm.entries[prefix] = e
|
||||
}
|
||||
|
||||
logCallerF("Increasing allowed IP ref count for prefix %v peer %s [peer %d -> %d, total %d -> %d, active %q]",
|
||||
prefix, peerKey, e.peers[peerKey], e.peers[peerKey]+1, e.total, e.total+1, e.active)
|
||||
|
||||
// Program WireGuard only when nothing is installed yet for this prefix.
|
||||
if e.active == "" {
|
||||
out, err := rm.add(prefix, peerKey)
|
||||
if errors.Is(err, ErrIgnore) {
|
||||
if e.total == 0 {
|
||||
delete(rm.entries, prefix)
|
||||
}
|
||||
return Ref[string]{Count: e.total, Out: e.active}, nil
|
||||
}
|
||||
if err != nil {
|
||||
if e.total == 0 {
|
||||
delete(rm.entries, prefix)
|
||||
}
|
||||
return Ref[string]{}, fmt.Errorf("failed to add allowed IP %v for peer %s: %w", prefix, peerKey, err)
|
||||
}
|
||||
e.active = out
|
||||
}
|
||||
|
||||
e.peers[peerKey]++
|
||||
e.total++
|
||||
|
||||
return Ref[string]{Count: e.total, Out: e.active}, nil
|
||||
}
|
||||
|
||||
// Decrement removes a reference to prefix for peerKey. When the peer currently installed in
|
||||
// WireGuard releases its last reference, the prefix is swapped to a surviving peer if one exists,
|
||||
// otherwise it is removed from WireGuard.
|
||||
func (rm *AllowedIPsRefCounter) Decrement(prefix netip.Prefix, peerKey string) (Ref[string], error) {
|
||||
rm.mu.Lock()
|
||||
defer rm.mu.Unlock()
|
||||
|
||||
e, ok := rm.entries[prefix]
|
||||
if !ok {
|
||||
logCallerF("No allowed IP reference found for prefix %v", prefix)
|
||||
return Ref[string]{}, nil
|
||||
}
|
||||
|
||||
if e.peers[peerKey] > 0 {
|
||||
logCallerF("Decreasing allowed IP ref count for prefix %v peer %s [peer %d -> %d, total %d -> %d, active %q]",
|
||||
prefix, peerKey, e.peers[peerKey], e.peers[peerKey]-1, e.total, e.total-1, e.active)
|
||||
e.peers[peerKey]--
|
||||
e.total--
|
||||
if e.peers[peerKey] == 0 {
|
||||
delete(e.peers, peerKey)
|
||||
}
|
||||
} else {
|
||||
logCallerF("No allowed IP reference found for prefix %v peer %s", prefix, peerKey)
|
||||
}
|
||||
|
||||
// If the peer currently installed in WireGuard still holds references, nothing to reprogram.
|
||||
// Keying the check on the active peer (not the one just released) makes this self-healing:
|
||||
// a prior swap whose remove/add failed leaves e.active pointing at a peer with no references,
|
||||
// and this retries the hand-off on the next Decrement instead of getting stuck.
|
||||
if e.active != "" && e.peers[e.active] > 0 {
|
||||
return Ref[string]{Count: e.total, Out: e.active}, nil
|
||||
}
|
||||
|
||||
// Detach the stale/gone active peer from WireGuard before reprogramming.
|
||||
if e.active != "" {
|
||||
if err := rm.remove(prefix, e.active); err != nil {
|
||||
return Ref[string]{Count: e.total, Out: e.active}, fmt.Errorf("remove allowed IP %v for peer %s: %w", prefix, e.active, err)
|
||||
}
|
||||
e.active = ""
|
||||
}
|
||||
|
||||
// Hand the prefix over to a surviving peer, or drop the entry when none remain.
|
||||
if survivor, ok := pickSurvivor(e.peers); ok {
|
||||
out, err := rm.add(prefix, survivor)
|
||||
if err != nil {
|
||||
return Ref[string]{Count: e.total, Out: ""}, fmt.Errorf("swap allowed IP %v to peer %s: %w", prefix, survivor, err)
|
||||
}
|
||||
e.active = out
|
||||
return Ref[string]{Count: e.total, Out: e.active}, nil
|
||||
}
|
||||
|
||||
delete(rm.entries, prefix)
|
||||
return Ref[string]{Count: 0, Out: ""}, nil
|
||||
}
|
||||
|
||||
// Flush removes all prefixes from WireGuard and clears the counter.
|
||||
func (rm *AllowedIPsRefCounter) Flush() error {
|
||||
rm.mu.Lock()
|
||||
defer rm.mu.Unlock()
|
||||
|
||||
var merr *multierror.Error
|
||||
for prefix, e := range rm.entries {
|
||||
if e.active == "" {
|
||||
continue
|
||||
}
|
||||
logCallerF("Flushing allowed IP for prefix %v peer %s", prefix, e.active)
|
||||
if err := rm.remove(prefix, e.active); err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %v for peer %s: %w", prefix, e.active, err))
|
||||
}
|
||||
}
|
||||
|
||||
clear(rm.entries)
|
||||
|
||||
return nberrors.FormatErrorOrNil(merr)
|
||||
}
|
||||
|
||||
// pickSurvivor deterministically selects a peer still referencing the prefix. WireGuard cannot do
|
||||
// multipath for a single prefix, so any surviving peer is a valid winner; the choice is made stable
|
||||
// (lowest peerKey) for predictable behavior and testability.
|
||||
func pickSurvivor(peers map[string]int) (string, bool) {
|
||||
if len(peers) == 0 {
|
||||
return "", false
|
||||
}
|
||||
keys := make([]string, 0, len(peers))
|
||||
for k := range peers {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys[0], true
|
||||
}
|
||||
241
client/internal/routemanager/refcounter/allowedips_test.go
Normal file
241
client/internal/routemanager/refcounter/allowedips_test.go
Normal file
@@ -0,0 +1,241 @@
|
||||
package refcounter
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/netip"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeWG models WireGuard's cryptokey routing: a prefix can be installed on exactly one peer.
|
||||
// failAdd/failRemove make the next add/remove fail once, to exercise the self-healing error paths.
|
||||
type fakeWG struct {
|
||||
installed map[netip.Prefix]string
|
||||
adds int
|
||||
removes int
|
||||
failAdd bool
|
||||
failRemove bool
|
||||
}
|
||||
|
||||
func newFakeWG() *fakeWG {
|
||||
return &fakeWG{installed: map[netip.Prefix]string{}}
|
||||
}
|
||||
|
||||
func (f *fakeWG) counter() *AllowedIPsRefCounter {
|
||||
return NewAllowedIPs(
|
||||
func(prefix netip.Prefix, peerKey string) (string, error) {
|
||||
if f.failAdd {
|
||||
f.failAdd = false
|
||||
return "", errors.New("add failed")
|
||||
}
|
||||
f.adds++
|
||||
f.installed[prefix] = peerKey
|
||||
return peerKey, nil
|
||||
},
|
||||
func(prefix netip.Prefix, peerKey string) error {
|
||||
if f.failRemove {
|
||||
f.failRemove = false
|
||||
return errors.New("remove failed")
|
||||
}
|
||||
f.removes++
|
||||
// only clear if this peer is the one installed, mirroring wg semantics
|
||||
if f.installed[prefix] == peerKey {
|
||||
delete(f.installed, prefix)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func mustPrefix(t *testing.T, s string) netip.Prefix {
|
||||
t.Helper()
|
||||
p, err := netip.ParsePrefix(s)
|
||||
if err != nil {
|
||||
t.Fatalf("parse prefix %q: %v", s, err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func mustIncrement(t *testing.T, c *AllowedIPsRefCounter, p netip.Prefix, peer string) Ref[string] {
|
||||
t.Helper()
|
||||
ref, err := c.Increment(p, peer)
|
||||
if err != nil {
|
||||
t.Fatalf("Increment(%v, %s): %v", p, peer, err)
|
||||
}
|
||||
return ref
|
||||
}
|
||||
|
||||
func mustDecrement(t *testing.T, c *AllowedIPsRefCounter, p netip.Prefix, peer string) Ref[string] {
|
||||
t.Helper()
|
||||
ref, err := c.Decrement(p, peer)
|
||||
if err != nil {
|
||||
t.Fatalf("Decrement(%v, %s): %v", p, peer, err)
|
||||
}
|
||||
return ref
|
||||
}
|
||||
|
||||
// TestAllowedIPs_SwapOnActivePeerRemoval reproduces the reported bug: two networks with the same
|
||||
// prefix routed by different peers. Removing the network whose peer is installed must hand the
|
||||
// prefix over to the surviving peer instead of leaving it on the removed one.
|
||||
func TestAllowedIPs_SwapOnActivePeerRemoval(t *testing.T) {
|
||||
f := newFakeWG()
|
||||
c := f.counter()
|
||||
p := mustPrefix(t, "10.44.8.0/24")
|
||||
|
||||
mustIncrement(t, c, p, "peerA")
|
||||
mustIncrement(t, c, p, "peerB")
|
||||
// First peer wins while both are present.
|
||||
if got := f.installed[p]; got != "peerA" {
|
||||
t.Fatalf("expected peerA installed, got %q", got)
|
||||
}
|
||||
|
||||
// Remove the active peer's network -> must swap to peerB.
|
||||
mustDecrement(t, c, p, "peerA")
|
||||
if got := f.installed[p]; got != "peerB" {
|
||||
t.Fatalf("BUG: prefix stuck on removed peer, want peerB got %q", got)
|
||||
}
|
||||
|
||||
// Remove the last one -> prefix gone.
|
||||
mustDecrement(t, c, p, "peerB")
|
||||
if _, ok := f.installed[p]; ok {
|
||||
t.Fatalf("expected prefix removed, still installed on %q", f.installed[p])
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowedIPs_RemoveNonActivePeer removing a non-installed peer must not touch WireGuard.
|
||||
func TestAllowedIPs_RemoveNonActivePeer(t *testing.T) {
|
||||
f := newFakeWG()
|
||||
c := f.counter()
|
||||
p := mustPrefix(t, "10.44.8.0/24")
|
||||
|
||||
mustIncrement(t, c, p, "peerA")
|
||||
mustIncrement(t, c, p, "peerB")
|
||||
removesBefore := f.removes
|
||||
|
||||
mustDecrement(t, c, p, "peerB")
|
||||
if f.installed[p] != "peerA" {
|
||||
t.Fatalf("active peer must stay peerA, got %q", f.installed[p])
|
||||
}
|
||||
if f.removes != removesBefore {
|
||||
t.Fatalf("removing a non-active peer must not call wg remove")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowedIPs_SamePeerMultipleRefs two references via the same peer must keep the prefix until
|
||||
// the last reference is released (the reason the per-peer count must be an int, not a set).
|
||||
func TestAllowedIPs_SamePeerMultipleRefs(t *testing.T) {
|
||||
f := newFakeWG()
|
||||
c := f.counter()
|
||||
p := mustPrefix(t, "10.44.8.0/24")
|
||||
|
||||
mustIncrement(t, c, p, "peerA")
|
||||
mustIncrement(t, c, p, "peerA")
|
||||
if f.adds != 1 {
|
||||
t.Fatalf("expected a single wg add for the same peer, got %d", f.adds)
|
||||
}
|
||||
|
||||
mustDecrement(t, c, p, "peerA")
|
||||
if f.installed[p] != "peerA" {
|
||||
t.Fatalf("prefix must stay while a reference remains, got %q", f.installed[p])
|
||||
}
|
||||
if f.removes != 0 {
|
||||
t.Fatalf("no wg remove expected while a reference remains, got %d", f.removes)
|
||||
}
|
||||
|
||||
mustDecrement(t, c, p, "peerA")
|
||||
if _, ok := f.installed[p]; ok {
|
||||
t.Fatalf("prefix must be removed after last reference")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowedIPs_RefCountAndActive checks the Ref returned to callers (used for the HA-disabled log).
|
||||
func TestAllowedIPs_RefCountAndActive(t *testing.T) {
|
||||
f := newFakeWG()
|
||||
c := f.counter()
|
||||
p := mustPrefix(t, "10.44.8.0/24")
|
||||
|
||||
ref := mustIncrement(t, c, p, "peerA")
|
||||
if ref.Count != 1 || ref.Out != "peerA" {
|
||||
t.Fatalf("want {1, peerA}, got {%d, %q}", ref.Count, ref.Out)
|
||||
}
|
||||
ref = mustIncrement(t, c, p, "peerB")
|
||||
if ref.Count != 2 || ref.Out != "peerA" {
|
||||
t.Fatalf("want {2, peerA}, got {%d, %q}", ref.Count, ref.Out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowedIPs_Flush removes everything installed and clears the counter.
|
||||
func TestAllowedIPs_Flush(t *testing.T) {
|
||||
f := newFakeWG()
|
||||
c := f.counter()
|
||||
p1 := mustPrefix(t, "10.44.8.0/24")
|
||||
p2 := mustPrefix(t, "10.44.9.0/24")
|
||||
|
||||
mustIncrement(t, c, p1, "peerA")
|
||||
mustIncrement(t, c, p2, "peerB")
|
||||
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.installed) != 0 {
|
||||
t.Fatalf("expected all prefixes removed, got %v", f.installed)
|
||||
}
|
||||
// After flush, a fresh increment must add again.
|
||||
mustIncrement(t, c, p1, "peerC")
|
||||
if f.installed[p1] != "peerC" {
|
||||
t.Fatalf("counter not reset after flush")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowedIPs_SelfHealAfterSwapAddError ensures a failed add during a swap does not permanently
|
||||
// strand the prefix: the next Decrement (or Increment) must retry and install a surviving peer.
|
||||
func TestAllowedIPs_SelfHealAfterSwapAddError(t *testing.T) {
|
||||
f := newFakeWG()
|
||||
c := f.counter()
|
||||
p := mustPrefix(t, "10.44.8.0/24")
|
||||
|
||||
mustIncrement(t, c, p, "peerA")
|
||||
mustIncrement(t, c, p, "peerB")
|
||||
mustIncrement(t, c, p, "peerC")
|
||||
|
||||
// Removing the active peerA triggers a swap to a survivor; make the add fail once.
|
||||
f.failAdd = true
|
||||
if _, err := c.Decrement(p, "peerA"); err == nil {
|
||||
t.Fatalf("expected error from failed swap add")
|
||||
}
|
||||
if _, ok := f.installed[p]; ok {
|
||||
t.Fatalf("nothing should be installed after a failed swap add, got %q", f.installed[p])
|
||||
}
|
||||
|
||||
// A later Decrement of a non-active survivor must retry the hand-off (self-heal), not stay stuck.
|
||||
ref := mustDecrement(t, c, p, "peerC")
|
||||
if got := f.installed[p]; got == "" {
|
||||
t.Fatalf("self-heal failed: prefix left unrouted after add recovered")
|
||||
}
|
||||
if ref.Out == "" {
|
||||
t.Fatalf("expected an active peer after self-heal, got empty")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowedIPs_SelfHealAfterRemoveError ensures a failed remove during a swap is retried instead
|
||||
// of leaving e.active stuck on a peer that no longer holds references.
|
||||
func TestAllowedIPs_SelfHealAfterRemoveError(t *testing.T) {
|
||||
f := newFakeWG()
|
||||
c := f.counter()
|
||||
p := mustPrefix(t, "10.44.8.0/24")
|
||||
|
||||
mustIncrement(t, c, p, "peerA")
|
||||
mustIncrement(t, c, p, "peerB")
|
||||
|
||||
// Releasing active peerA must detach it (remove) then add peerB; fail the remove once.
|
||||
f.failRemove = true
|
||||
if _, err := c.Decrement(p, "peerA"); err == nil {
|
||||
t.Fatalf("expected error from failed remove")
|
||||
}
|
||||
|
||||
// Next Decrement of the non-active survivor retries: removes stale peerA, installs peerB.
|
||||
mustDecrement(t, c, p, "peerB")
|
||||
// peerB had only one ref, so after retry the prefix is fully released.
|
||||
if _, ok := f.installed[p]; ok {
|
||||
t.Fatalf("expected prefix released after self-heal, still on %q", f.installed[p])
|
||||
}
|
||||
}
|
||||
@@ -5,5 +5,7 @@ import "net/netip"
|
||||
// RouteRefCounter is a Counter for Route, it doesn't take any input on Increment and doesn't use any output on Decrement
|
||||
type RouteRefCounter = Counter[netip.Prefix, struct{}, struct{}]
|
||||
|
||||
// AllowedIPsRefCounter is a Counter for AllowedIPs, it takes a peer key on Increment and passes it back to Decrement
|
||||
type AllowedIPsRefCounter = Counter[netip.Prefix, string, string]
|
||||
// AllowedIPsRefCounter tracks WireGuard AllowedIPs per prefix. Unlike the generic Counter it is peer-aware:
|
||||
// a prefix can be claimed by several peers at once and WireGuard allows a given prefix on exactly one peer,
|
||||
// so the counter records the per-peer reference count and swaps the installed peer when the active one is released.
|
||||
// See allowedips.go.
|
||||
|
||||
@@ -15,6 +15,11 @@ type Route struct {
|
||||
route *route.Route
|
||||
routeRefCounter *refcounter.RouteRefCounter
|
||||
allowedIPsRefcounter *refcounter.AllowedIPsRefCounter
|
||||
// currentPeerKey is the routing peer this watcher currently has the prefix installed on
|
||||
// (the HA winner elected by the watcher). It can differ from route.Peer and change on
|
||||
// failover, so it is recorded on AddAllowedIPs and used on RemoveAllowedIPs to decrement
|
||||
// the exact peer that was incremented.
|
||||
currentPeerKey string
|
||||
}
|
||||
|
||||
func NewRoute(params common.HandlerParams) *Route {
|
||||
@@ -52,12 +57,15 @@ func (r *Route) AddAllowedIPs(peerKey string) error {
|
||||
ref.Out,
|
||||
)
|
||||
}
|
||||
r.currentPeerKey = peerKey
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Route) RemoveAllowedIPs() error {
|
||||
if _, err := r.allowedIPsRefcounter.Decrement(r.route.Network); err != nil {
|
||||
return err
|
||||
var err error
|
||||
if _, decErr := r.allowedIPsRefcounter.Decrement(r.route.Network, r.currentPeerKey); decErr != nil {
|
||||
err = fmt.Errorf("remove allowed IP %s: %w", r.route.Network, decErr)
|
||||
}
|
||||
return nil
|
||||
r.currentPeerKey = ""
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/netbirdio/netbird/client/internal/listener"
|
||||
"github.com/netbirdio/netbird/client/internal/peer"
|
||||
"github.com/netbirdio/netbird/client/internal/profilemanager"
|
||||
"github.com/netbirdio/netbird/client/mdm"
|
||||
"github.com/netbirdio/netbird/client/system"
|
||||
"github.com/netbirdio/netbird/formatter"
|
||||
"github.com/netbirdio/netbird/route"
|
||||
@@ -83,6 +84,13 @@ type Client struct {
|
||||
// preloadedConfig holds config loaded from JSON (used on tvOS where file writes are blocked)
|
||||
preloadedConfig *profilemanager.Config
|
||||
|
||||
// mdmLoader holds the per-Client MDM policy source. Set by
|
||||
// SetMDMPolicyFetcher (called from the Swift side at extension
|
||||
// init). Each Run passes this loader to the resolved Config so
|
||||
// applyMDMPolicy picks up the active overlay. Nil means "MDM
|
||||
// enforcement off for this Client".
|
||||
mdmLoader *mdm.Loader
|
||||
|
||||
stateMu sync.RWMutex
|
||||
connectClient *internal.ConnectClient
|
||||
config *profilemanager.Config
|
||||
@@ -143,6 +151,7 @@ func (c *Client) Run(fd int32, interfaceName string, envList *EnvList) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.applyMDMOverlay(cfg)
|
||||
}
|
||||
c.recorder.UpdateManagementAddress(cfg.ManagementURL.String())
|
||||
c.recorder.UpdateRosenpass(cfg.RosenpassEnabled, cfg.RosenpassPermissive)
|
||||
@@ -215,6 +224,7 @@ func (c *Client) DebugBundle(anonymize bool) (string, error) {
|
||||
return "", fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
}
|
||||
c.applyMDMOverlay(cfg)
|
||||
}
|
||||
|
||||
deps := debug.GeneratorDependencies{
|
||||
@@ -364,6 +374,7 @@ func (c *Client) IsLoginRequired() bool {
|
||||
// If we can't load config, assume login is required
|
||||
return true
|
||||
}
|
||||
c.applyMDMOverlay(cfg)
|
||||
}
|
||||
|
||||
if cfg == nil {
|
||||
@@ -412,6 +423,7 @@ func (c *Client) LoginForMobile() string {
|
||||
log.Errorf("LoginForMobile: failed to load config: %v", err)
|
||||
return fmt.Sprintf("failed to load config: %v", err)
|
||||
}
|
||||
c.applyMDMOverlay(cfg)
|
||||
|
||||
oAuthFlow, err := auth.NewOAuthFlow(ctx, cfg, false, false, "")
|
||||
if err != nil {
|
||||
|
||||
82
client/ios/NetBirdSDK/mdm.go
Normal file
82
client/ios/NetBirdSDK/mdm.go
Normal file
@@ -0,0 +1,82 @@
|
||||
//go:build ios
|
||||
|
||||
package NetBirdSDK
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/netbirdio/netbird/client/internal/profilemanager"
|
||||
"github.com/netbirdio/netbird/client/mdm"
|
||||
)
|
||||
|
||||
// PolicyFetcher is the mobile-side bridge for the MDM managed-config
|
||||
// snapshot. The native layer (Swift) implements this and registers
|
||||
// the instance per Client via Client.SetMDMPolicyFetcher. Every
|
||||
// invocation of fetchJSON must read the current
|
||||
// UserDefaults.standard.dictionary(forKey: "com.apple.configuration.managed")
|
||||
// and return the result as a JSON-encoded map[string]any string.
|
||||
//
|
||||
// JSON is used because gomobile does not support map[string]any
|
||||
// crossing the Objective-C boundary — the adapter on the Go side
|
||||
// parses the string back into the map[string]any expected by
|
||||
// mdm.Loader.
|
||||
//
|
||||
// Return value contract:
|
||||
// - "" (empty) : interpreted as "no MDM source / no managed keys"
|
||||
// - "{}" : managed config explicitly empty
|
||||
// - "{...}" : JSON object with key/value pairs
|
||||
// - malformed JSON : logged and treated as empty
|
||||
type PolicyFetcher interface {
|
||||
FetchJSON() string
|
||||
}
|
||||
|
||||
// jsonFetcherAdapter wraps a gomobile-exposed PolicyFetcher into the
|
||||
// internal mdm.PolicyFetcher interface, taking care of JSON decoding
|
||||
// on every Fetch.
|
||||
type jsonFetcherAdapter struct {
|
||||
inner PolicyFetcher
|
||||
}
|
||||
|
||||
func (a *jsonFetcherAdapter) Fetch() map[string]any {
|
||||
raw := a.inner.FetchJSON()
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal([]byte(raw), &out); err != nil {
|
||||
log.Warnf("MDM mobile fetcher: invalid JSON payload from native: %v", err)
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// SetMDMPolicyFetcher registers the native-provided MDM policy fetcher
|
||||
// on this Client. Call once from the gomobile-init code (Swift
|
||||
// AppDelegate / PacketTunnelProvider.startTunnel) before invoking Run.
|
||||
// Passing nil disables MDM enforcement on this Client.
|
||||
//
|
||||
// The fetcher is held as a *mdm.Loader instance on the Client (no
|
||||
// package-level state) — multiple Clients in the same process get
|
||||
// independent Loaders, and tests can inject fakes per Client.
|
||||
func (c *Client) SetMDMPolicyFetcher(p PolicyFetcher) {
|
||||
if p == nil {
|
||||
c.mdmLoader = mdm.NewLoader(nil)
|
||||
return
|
||||
}
|
||||
c.mdmLoader = mdm.NewLoader(&jsonFetcherAdapter{inner: p})
|
||||
}
|
||||
|
||||
// applyMDMOverlay applies the Client-held MDM Loader's current policy
|
||||
// on top of the just-read Config. Called immediately after every
|
||||
// UpdateOrCreateConfig / DirectUpdateOrCreateConfig — profilemanager's
|
||||
// apply() initialises the policy to empty and leaves overlay
|
||||
// responsibility to the lifecycle owner. No-op when no fetcher was
|
||||
// registered.
|
||||
func (c *Client) applyMDMOverlay(cfg *profilemanager.Config) {
|
||||
if cfg == nil || c.mdmLoader == nil {
|
||||
return
|
||||
}
|
||||
cfg.ApplyMDMPolicy(c.mdmLoader.Load())
|
||||
}
|
||||
12
client/ios/NetBirdSDK/version.go
Normal file
12
client/ios/NetBirdSDK/version.go
Normal file
@@ -0,0 +1,12 @@
|
||||
//go:build ios
|
||||
|
||||
package NetBirdSDK
|
||||
|
||||
import "github.com/netbirdio/netbird/version"
|
||||
|
||||
// GoClientVersion returns the NetBird Go client version that was baked into
|
||||
// the framework at compile time via
|
||||
// -ldflags "-X github.com/netbirdio/netbird/version.version=<version>".
|
||||
func GoClientVersion() string {
|
||||
return version.NetbirdVersion()
|
||||
}
|
||||
@@ -104,16 +104,48 @@ func NewPolicy(values map[string]any) *Policy {
|
||||
return &Policy{values: values}
|
||||
}
|
||||
|
||||
// LoadPolicy reads the platform-native MDM configuration. Returns an
|
||||
// empty (but non-nil) Policy when no source is present, the source is
|
||||
// empty, or the platform is unsupported.
|
||||
// PolicyFetcher is implemented by mobile platforms (Android / iOS) that
|
||||
// push the OS-managed configuration into the Go runtime instead of
|
||||
// having Go read an on-disk source directly. Desktop platforms ignore
|
||||
// this interface — Loader.loadPlatform on windows/darwin reads the
|
||||
// registry / plist on its own. A Loader constructed with a non-nil
|
||||
// fetcher delegates to it on mobile; passing nil disables MDM
|
||||
// enforcement (loadPlatform returns nil values).
|
||||
type PolicyFetcher interface {
|
||||
Fetch() map[string]any
|
||||
}
|
||||
|
||||
// Loader is the DI-friendly entry point for reading the active MDM
|
||||
// policy. Construct one at the daemon's lifecycle owner (Server on
|
||||
// desktop, gomobile-exposed bridge on mobile) and pass it to anything
|
||||
// that needs to read MDM state (the reload ticker, profilemanager's
|
||||
// Config). Each callsite has the Loader handed in instead of looking
|
||||
// up package-level state.
|
||||
type Loader struct {
|
||||
fetcher PolicyFetcher
|
||||
}
|
||||
|
||||
// NewLoader constructs a Loader. The fetcher is consulted only on
|
||||
// mobile builds (ios || android); on desktop it is unused but accepted
|
||||
// to keep a single constructor signature across platforms — pass nil
|
||||
// on desktop.
|
||||
func NewLoader(f PolicyFetcher) *Loader {
|
||||
return &Loader{fetcher: f}
|
||||
}
|
||||
|
||||
// Load reads the platform-native MDM configuration and returns a
|
||||
// Policy. Returns an empty (but non-nil) Policy when no source is
|
||||
// present, the source is empty, or the platform is unsupported.
|
||||
//
|
||||
// Diagnostic logging differentiates the three states:
|
||||
// - source absent / unsupported platform: trace log only
|
||||
// - source present, zero keys: info "MDM enrolled (no managed keys)"
|
||||
// - source present, N keys: info "MDM enrolled with N managed keys: [...]"
|
||||
func LoadPolicy() *Policy {
|
||||
values, err := loadPlatformPolicy()
|
||||
func (l *Loader) Load() *Policy {
|
||||
if l == nil {
|
||||
return &Policy{values: map[string]any{}}
|
||||
}
|
||||
values, err := l.loadPlatform()
|
||||
if err != nil {
|
||||
log.Tracef("MDM policy load: %v", err)
|
||||
return &Policy{values: map[string]any{}}
|
||||
|
||||
@@ -25,8 +25,10 @@ import (
|
||||
// writable plist, as a defense against tampered installs.
|
||||
const policyPlistPath = "/Library/Managed Preferences/io.netbird.client.plist"
|
||||
|
||||
// loadPlatformPolicy reads the MDM-managed configuration from the macOS
|
||||
// managed-preferences plist at policyPlistPath. Returns:
|
||||
// loadPlatform reads the MDM-managed configuration from the macOS
|
||||
// managed-preferences plist at policyPlistPath. The Loader's fetcher
|
||||
// field is unused on this platform — the plist is the authoritative
|
||||
// source. Returns:
|
||||
// - (nil, nil) when the plist is absent (device not MDM-enrolled for
|
||||
// NetBird, or admin has not yet pushed a payload)
|
||||
// - (map, nil) with N entries when N managed values are present
|
||||
@@ -39,7 +41,13 @@ const policyPlistPath = "/Library/Managed Preferences/io.netbird.client.plist"
|
||||
// skipped so a stray entry in the payload does not block startup.
|
||||
// Native plist value types map naturally onto the Policy accessor
|
||||
// expectations (GetString / GetBool / GetInt / GetStringSlice).
|
||||
func loadPlatformPolicy() (map[string]any, error) {
|
||||
func (l *Loader) loadPlatform() (map[string]any, error) {
|
||||
// Honour the injected fetcher when present so tests (and any
|
||||
// future non-macOS MDM channel) can short-circuit the plist read
|
||||
// with a scripted policy.
|
||||
if l != nil && l.fetcher != nil {
|
||||
return l.fetcher.Fetch(), nil
|
||||
}
|
||||
f, err := os.Open(policyPlistPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
|
||||
@@ -2,13 +2,14 @@
|
||||
|
||||
package mdm
|
||||
|
||||
// loadPlatformPolicy is unused on mobile: the native layer (Swift on iOS,
|
||||
// Kotlin/Java on Android) reads the OS managed-config store and pushes the
|
||||
// resulting dictionary in-process via a gomobile entry point that lands in
|
||||
// Phase 5 / Phase 6. The stub keeps the package compilable for mobile
|
||||
// builds and returns (nil, nil) — the platform-absent sentinel that
|
||||
// LoadPolicy in policy.go treats as "no MDM source present".
|
||||
func loadPlatformPolicy() (map[string]any, error) {
|
||||
//nolint:nilnil // (nil, nil) is the documented platform-absent sentinel; see LoadPolicy.
|
||||
return nil, nil
|
||||
// loadPlatform reads the OS-managed configuration via the native
|
||||
// PolicyFetcher injected at Loader construction. Returns
|
||||
// (nil, nil) — the platform-absent sentinel that Loader.Load treats as
|
||||
// "no MDM source present" — when no fetcher was provided.
|
||||
func (l *Loader) loadPlatform() (map[string]any, error) {
|
||||
if l == nil || l.fetcher == nil {
|
||||
//nolint:nilnil // (nil, nil) is the documented platform-absent sentinel; see Loader.Load.
|
||||
return nil, nil
|
||||
}
|
||||
return l.fetcher.Fetch(), nil
|
||||
}
|
||||
|
||||
@@ -2,13 +2,17 @@
|
||||
|
||||
package mdm
|
||||
|
||||
// loadPlatformPolicy returns no policy on platforms without an MDM channel
|
||||
// (Linux, FreeBSD). MDM enforcement is off and the client behaves as if
|
||||
// the feature did not exist. Returns (nil, nil) — the platform-absent
|
||||
// sentinel the caller (LoadPolicy in policy.go) treats as "no MDM
|
||||
// source present"; an error here would just translate to the same
|
||||
// outcome with an extra log line.
|
||||
func loadPlatformPolicy() (map[string]any, error) {
|
||||
//nolint:nilnil // (nil, nil) is the documented platform-absent sentinel; see LoadPolicy.
|
||||
// loadPlatform reads the MDM policy on platforms without a native MDM
|
||||
// channel (Linux, FreeBSD). When no fetcher was injected the policy is
|
||||
// (nil, nil) — the platform-absent sentinel that Loader.Load treats as
|
||||
// "MDM enforcement disabled". A non-nil fetcher takes precedence: it
|
||||
// is the test-seam used by unit tests to inject a scripted policy
|
||||
// without touching the OS, and the same hook supports any future
|
||||
// non-mobile OS that grows an out-of-band MDM channel.
|
||||
func (l *Loader) loadPlatform() (map[string]any, error) {
|
||||
if l != nil && l.fetcher != nil {
|
||||
return l.fetcher.Fetch(), nil
|
||||
}
|
||||
//nolint:nilnil // (nil, nil) is the documented platform-absent sentinel; see Loader.Load.
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -155,10 +155,12 @@ func TestPolicy_GetStringSlice(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestLoadPolicy_PlatformStubReturnsEmpty(t *testing.T) {
|
||||
// loadPlatformPolicy is a stub on every OS for Phase 1. LoadPolicy must
|
||||
// degrade gracefully and never return nil.
|
||||
p := LoadPolicy()
|
||||
func TestLoader_NilFetcherReturnsEmpty(t *testing.T) {
|
||||
// Loader.Load with no fetcher (desktop construction) must degrade
|
||||
// gracefully and never return nil; on linux loadPlatform is a stub
|
||||
// returning (nil, nil), and Load is expected to translate that
|
||||
// into a non-nil empty Policy.
|
||||
p := NewLoader(nil).Load()
|
||||
require.NotNil(t, p)
|
||||
assert.True(t, p.IsEmpty())
|
||||
assert.Empty(t, p.ManagedKeys())
|
||||
|
||||
@@ -61,8 +61,10 @@ func readRegistryValue(k registry.Key, name, canonical string, out map[string]an
|
||||
}
|
||||
}
|
||||
|
||||
// loadPlatformPolicy reads the MDM-managed configuration from the
|
||||
// Windows registry under HKLM\Software\Policies\NetBird. Returns:
|
||||
// loadPlatform reads the MDM-managed configuration from the Windows
|
||||
// registry under HKLM\Software\Policies\NetBird. The Loader's fetcher
|
||||
// field is unused on this platform — the registry is the
|
||||
// authoritative source. Returns:
|
||||
// - (nil, nil) when the key is absent (device not MDM-enrolled for NetBird)
|
||||
// - (map, nil) with N entries when N managed values are set (N may be 0)
|
||||
// - (nil, err) on open / enumerate registry errors
|
||||
@@ -70,7 +72,13 @@ func readRegistryValue(k registry.Key, name, canonical string, out map[string]an
|
||||
// Per-value type coercion + skip-on-error is delegated to
|
||||
// readRegistryValue. Unknown value names are logged and skipped so a
|
||||
// malformed deployment does not block startup.
|
||||
func loadPlatformPolicy() (map[string]any, error) {
|
||||
func (l *Loader) loadPlatform() (map[string]any, error) {
|
||||
// Honour the injected fetcher when present so tests (and any
|
||||
// future non-Windows MDM channel) can short-circuit the registry
|
||||
// read with a scripted policy.
|
||||
if l != nil && l.fetcher != nil {
|
||||
return l.fetcher.Fetch(), nil
|
||||
}
|
||||
k, err := registry.OpenKey(registry.LOCAL_MACHINE, policyRegistryPath, registry.QUERY_VALUE)
|
||||
if err != nil {
|
||||
if errors.Is(err, registry.ErrNotExist) {
|
||||
|
||||
@@ -15,33 +15,33 @@ import (
|
||||
// instead, hence anticipating the ticker mechanism entirely.
|
||||
const DefaultReloadInterval = 1 * time.Minute
|
||||
|
||||
// policyLoader is the indirection through which the ticker reads the
|
||||
// OS-native policy, both for the initial observation and on every tick.
|
||||
// Production points it at LoadPolicy; tests in this package override it to
|
||||
// feed a scripted sequence of policies without touching the real OS store.
|
||||
var policyLoader = LoadPolicy
|
||||
|
||||
// Ticker periodically re-reads the OS-native MDM policy via LoadPolicy and
|
||||
// invokes the onChange callback (supplied to Run) whenever the observed
|
||||
// Policy diverges from the last observation (added / removed / changed
|
||||
// keys). Launch with Run from a goroutine; cancel the supplied context
|
||||
// to stop.
|
||||
// Ticker periodically re-reads the OS-native MDM policy via the
|
||||
// injected Loader and invokes the onChange callback (supplied to Run)
|
||||
// whenever the observed Policy diverges from the last observation
|
||||
// (added / removed / changed keys). Launch with Run from a goroutine;
|
||||
// cancel the supplied context to stop.
|
||||
type Ticker struct {
|
||||
interval time.Duration
|
||||
loader *Loader
|
||||
prev *Policy
|
||||
}
|
||||
|
||||
// NewTicker constructs a Ticker that will re-read the OS-native policy
|
||||
// every reloadInterval once Run is called.
|
||||
// The initial snapshot is populated by calling policyLoader at
|
||||
// every reloadInterval once Run is called. The Loader is injected so
|
||||
// the ticker doesn't depend on any package-level state — production
|
||||
// passes the daemon-owned Loader, tests pass a fake Loader (built with
|
||||
// a fake PolicyFetcher).
|
||||
//
|
||||
// The initial snapshot is populated by calling loader.Load() at
|
||||
// construction time so the first tick only fires
|
||||
// onChange when the policy actually changed since boot — without
|
||||
// this baseline the first tick would report every currently-managed
|
||||
// key as "added" and trigger a spurious engine restart.
|
||||
func NewTicker(reloadInterval time.Duration) *Ticker {
|
||||
func NewTicker(reloadInterval time.Duration, loader *Loader) *Ticker {
|
||||
return &Ticker{
|
||||
interval: reloadInterval,
|
||||
prev: policyLoader(),
|
||||
loader: loader,
|
||||
prev: loader.Load(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ func (t *Ticker) Run(ctx context.Context, onChange func(prev, curr *Policy) erro
|
||||
log.Info("MDM policy reload ticker stopped")
|
||||
return
|
||||
case <-tk.C:
|
||||
curr := policyLoader()
|
||||
curr := t.loader.Load()
|
||||
if policiesEqual(t.prev, curr) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -13,28 +13,40 @@ import (
|
||||
// testReloadInterval for speeding up the ticker cadence under `go test`
|
||||
const testReloadInterval = 1 * time.Second
|
||||
|
||||
// withPolicyLoader overrides the package-level policyLoader for the duration
|
||||
// of the test so the ticker observes a scripted policy instead of the real
|
||||
// OS-native store. The original loader is restored on cleanup.
|
||||
func withPolicyLoader(t *testing.T, fn func() *Policy) {
|
||||
t.Helper()
|
||||
prev := policyLoader
|
||||
policyLoader = fn
|
||||
t.Cleanup(func() { policyLoader = prev })
|
||||
// fakePolicyFetcher implements PolicyFetcher returning a scripted
|
||||
// policy map. Goroutine-safe so the test can mutate the script while
|
||||
// the ticker is observing it.
|
||||
type fakePolicyFetcher struct {
|
||||
mu sync.Mutex
|
||||
values map[string]any
|
||||
}
|
||||
|
||||
func (f *fakePolicyFetcher) Fetch() map[string]any {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.values == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(f.values))
|
||||
for k, v := range f.values {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (f *fakePolicyFetcher) set(values map[string]any) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.values = values
|
||||
}
|
||||
|
||||
func TestTicker_FiresOnChangeWithDelta(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
current := NewPolicy(nil) // initial observation: empty (no enforcement)
|
||||
withPolicyLoader(t, func() *Policy {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return current
|
||||
})
|
||||
fetcher := &fakePolicyFetcher{} // initial observation: empty (no enforcement)
|
||||
loader := NewLoader(fetcher)
|
||||
|
||||
type change struct{ prev, curr *Policy }
|
||||
changes := make(chan change, 1)
|
||||
tk := NewTicker(testReloadInterval)
|
||||
tk := NewTicker(testReloadInterval, loader)
|
||||
require.Equal(t, testReloadInterval, tk.interval)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
@@ -49,15 +61,13 @@ func TestTicker_FiresOnChangeWithDelta(t *testing.T) {
|
||||
})
|
||||
close(done)
|
||||
}()
|
||||
// Stop Run and wait for it to exit before returning, so the policyLoader
|
||||
// restore in t.Cleanup can't race the ticker goroutine still reading it.
|
||||
// Stop Run and wait for it to exit before returning, so the test
|
||||
// goroutine doesn't race the still-running ticker.
|
||||
defer func() { cancel(); <-done }()
|
||||
|
||||
// Flip the OS-observed policy from empty to one managed key. The next
|
||||
// tick must detect the diff and invoke onChange.
|
||||
mu.Lock()
|
||||
current = NewPolicy(map[string]any{KeyManagementURL: "https://mdm.example.com:443"})
|
||||
mu.Unlock()
|
||||
// Flip the OS-observed policy from empty to one managed key. The
|
||||
// next tick must detect the diff and invoke onChange.
|
||||
fetcher.set(map[string]any{KeyManagementURL: "https://mdm.example.com:443"})
|
||||
|
||||
select {
|
||||
case c := <-changes:
|
||||
@@ -69,12 +79,11 @@ func TestTicker_FiresOnChangeWithDelta(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTicker_NoCallbackWhenPolicyUnchanged(t *testing.T) {
|
||||
withPolicyLoader(t, func() *Policy {
|
||||
return NewPolicy(map[string]any{KeyBlockInbound: true})
|
||||
})
|
||||
fetcher := &fakePolicyFetcher{values: map[string]any{KeyBlockInbound: true}}
|
||||
loader := NewLoader(fetcher)
|
||||
|
||||
fired := make(chan struct{}, 1)
|
||||
tk := NewTicker(testReloadInterval)
|
||||
tk := NewTicker(testReloadInterval, loader)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
@@ -90,8 +99,8 @@ func TestTicker_NoCallbackWhenPolicyUnchanged(t *testing.T) {
|
||||
}()
|
||||
defer func() { cancel(); <-done }()
|
||||
|
||||
// Over ~2 ticks at the 1s test cadence the policy never changes, so the
|
||||
// diff guard must suppress the callback entirely.
|
||||
// Over ~2 ticks at the 1s test cadence the policy never changes,
|
||||
// so the diff guard must suppress the callback entirely.
|
||||
select {
|
||||
case <-fired:
|
||||
t.Fatal("onChange fired despite an unchanged policy")
|
||||
|
||||
@@ -21,10 +21,6 @@ import (
|
||||
// a no-op echo, never as a conflict with the policy.
|
||||
const preSharedKeyRedactedSentinel = "**********"
|
||||
|
||||
// loadMDMPolicy is the indirection used by server handlers to read the
|
||||
// active MDM policy. Tests override this to inject a fake policy.
|
||||
var loadMDMPolicy = mdm.LoadPolicy
|
||||
|
||||
// conflictCheck is a value-aware comparison between a single field in
|
||||
// the incoming request and the corresponding MDM-enforced value. It
|
||||
// runs only when the field was actually set in the request (presence
|
||||
|
||||
@@ -123,6 +123,15 @@ type Server struct {
|
||||
// stopped by the rootCtx cancellation.
|
||||
mdmTicker *mdm.Ticker
|
||||
|
||||
// mdmLoader is the daemon-owned source of the active MDM policy.
|
||||
// Constructed once during Server.Start (with a nil PolicyFetcher on
|
||||
// desktop — the build-tagged Loader.loadPlatform reads the OS
|
||||
// registry / plist directly) and injected into every consumer:
|
||||
// mdmTicker for its periodic reload, the SetConfig / Login MDM
|
||||
// gates for conflict detection, and every Config produced via
|
||||
// getConfig() so its apply() picks up the same overlay.
|
||||
mdmLoader *mdm.Loader
|
||||
|
||||
updateManager *updater.Manager
|
||||
|
||||
jwtCache *jwtCache
|
||||
@@ -197,8 +206,14 @@ func (s *Server) Start() error {
|
||||
// Runs re-resolves Config (re-running profilemanager.Config.apply which
|
||||
// applies the freshly-read MDM policy as the last layer) and brings
|
||||
// the engine back with the new values.
|
||||
if s.mdmLoader == nil {
|
||||
// Desktop builds pass a nil PolicyFetcher: the Loader's
|
||||
// build-tagged loadPlatform reads the OS source directly
|
||||
// (registry on Windows, plist on macOS, no-op elsewhere).
|
||||
s.mdmLoader = mdm.NewLoader(nil)
|
||||
}
|
||||
if s.mdmTicker == nil {
|
||||
s.mdmTicker = mdm.NewTicker(mdm.DefaultReloadInterval)
|
||||
s.mdmTicker = mdm.NewTicker(mdm.DefaultReloadInterval, s.mdmLoader)
|
||||
go s.mdmTicker.Run(s.rootCtx, s.onMDMPolicyChange)
|
||||
}
|
||||
|
||||
@@ -406,7 +421,7 @@ func (s *Server) SetConfig(callerCtx context.Context, msg *proto.SetConfigReques
|
||||
// by the active MDM policy. The error carries an MDMManagedFields-
|
||||
// Violation detail listing the offending key names. Non-conflicting
|
||||
// fields in the same request are not applied either.
|
||||
policy := loadMDMPolicy()
|
||||
policy := s.mdmLoader.Load()
|
||||
if err := rejectMDMManagedFieldConflicts(mdmManagedFieldConflicts(msg, policy)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -531,7 +546,7 @@ func (s *Server) Login(callerCtx context.Context, msg *proto.LoginRequest) (*pro
|
||||
if s.checkUpdateSettingsDisabled() {
|
||||
return nil, gstatus.Errorf(codes.Unavailable, errUpdateSettingsDisabled)
|
||||
}
|
||||
policy := loadMDMPolicy()
|
||||
policy := s.mdmLoader.Load()
|
||||
if err := rejectMDMManagedFieldConflicts(loginRequestMDMConflicts(msg, policy)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1260,6 +1275,12 @@ func (s *Server) getConfig(activeProf *profilemanager.ActiveProfileState) (*prof
|
||||
return nil, false, fmt.Errorf("failed to get config: %w", err)
|
||||
}
|
||||
|
||||
// Apply the daemon-owned MDM policy on top of the just-resolved
|
||||
// Config. profilemanager's apply() initialises the policy to
|
||||
// empty — the Loader lives outside Config, so this overlay step
|
||||
// is driven externally here.
|
||||
config.ApplyMDMPolicy(s.mdmLoader.Load())
|
||||
|
||||
return config, configExisted, nil
|
||||
}
|
||||
|
||||
@@ -1309,6 +1330,9 @@ func (s *Server) logoutFromProfile(ctx context.Context, profile *profilemanager.
|
||||
if err != nil {
|
||||
return fmt.Errorf("profile '%s' not found", profile.ID)
|
||||
}
|
||||
// Honour any MDM-enforced ManagementURL when issuing the logout
|
||||
// RPC: the user-stored value may have been overridden by policy.
|
||||
config.ApplyMDMPolicy(s.mdmLoader.Load())
|
||||
|
||||
return s.sendLogoutRequestWithConfig(ctx, config)
|
||||
}
|
||||
@@ -1909,6 +1933,11 @@ func (s *Server) GetConfig(ctx context.Context, req *proto.GetConfigRequest) (*p
|
||||
log.Errorf("failed to get active profile config: %v", err)
|
||||
return nil, fmt.Errorf("failed to get active profile config: %w", err)
|
||||
}
|
||||
// Overlay the active MDM policy so the response's MDMManagedFields
|
||||
// list reflects what the GUI / CLI must render as read-only.
|
||||
// profilemanager.GetConfig itself returns a Config without the
|
||||
// overlay (Loader lives outside profilemanager).
|
||||
cfg.ApplyMDMPolicy(s.mdmLoader.Load())
|
||||
managementURL := cfg.ManagementURL
|
||||
adminURL := cfg.AdminURL
|
||||
|
||||
|
||||
@@ -16,14 +16,40 @@ import (
|
||||
"github.com/netbirdio/netbird/client/proto"
|
||||
)
|
||||
|
||||
// withMDMPolicy temporarily overrides the server-package loadMDMPolicy hook
|
||||
// so SetConfig observes the supplied Policy. Restores the original loader
|
||||
// at test cleanup.
|
||||
func withMDMPolicy(t *testing.T, policy *mdm.Policy) {
|
||||
// fakeMDMFetcher implements mdm.PolicyFetcher returning a pre-set
|
||||
// policy map. Tests build one per Server instance to inject a
|
||||
// scripted MDM overlay via a Loader rather than via package-level state.
|
||||
type fakeMDMFetcher struct{ values map[string]any }
|
||||
|
||||
func (f *fakeMDMFetcher) Fetch() map[string]any { return f.values }
|
||||
|
||||
// withMDMPolicy installs an mdm.Loader on the given Server whose
|
||||
// loadPlatform returns the supplied Policy's underlying values. Use
|
||||
// after setupServerWithProfile to inject the scripted policy the
|
||||
// SetConfig / Login MDM gates will observe.
|
||||
func withMDMPolicy(t *testing.T, s *Server, policy *mdm.Policy) {
|
||||
t.Helper()
|
||||
prev := loadMDMPolicy
|
||||
loadMDMPolicy = func() *mdm.Policy { return policy }
|
||||
t.Cleanup(func() { loadMDMPolicy = prev })
|
||||
values := map[string]any{}
|
||||
if policy != nil {
|
||||
for _, k := range policy.ManagedKeys() {
|
||||
if v, ok := policy.GetString(k); ok {
|
||||
values[k] = v
|
||||
continue
|
||||
}
|
||||
if v, ok := policy.GetBool(k); ok {
|
||||
values[k] = v
|
||||
continue
|
||||
}
|
||||
if v, ok := policy.GetInt(k); ok {
|
||||
values[k] = v
|
||||
continue
|
||||
}
|
||||
if v, ok := policy.GetStringSlice(k); ok {
|
||||
values[k] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
s.mdmLoader = mdm.NewLoader(&fakeMDMFetcher{values: values})
|
||||
}
|
||||
|
||||
// setupServerWithProfile mirrors the boilerplate of TestSetConfig_AllFieldsSaved:
|
||||
@@ -89,12 +115,11 @@ func extractViolation(t *testing.T, err error) *proto.MDMManagedFieldsViolation
|
||||
}
|
||||
|
||||
func TestSetConfig_MDMReject_SingleField(t *testing.T) {
|
||||
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
|
||||
s, ctx, profName, username, _ := setupServerWithProfile(t)
|
||||
withMDMPolicy(t, s, mdm.NewPolicy(map[string]any{
|
||||
mdm.KeyManagementURL: "https://mdm.example.com:443",
|
||||
}))
|
||||
|
||||
s, ctx, profName, username, _ := setupServerWithProfile(t)
|
||||
|
||||
_, err := s.SetConfig(ctx, &proto.SetConfigRequest{
|
||||
ProfileName: profName,
|
||||
Username: username,
|
||||
@@ -106,14 +131,13 @@ func TestSetConfig_MDMReject_SingleField(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSetConfig_MDMReject_MultipleFields(t *testing.T) {
|
||||
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
|
||||
s, ctx, profName, username, _ := setupServerWithProfile(t)
|
||||
withMDMPolicy(t, s, mdm.NewPolicy(map[string]any{
|
||||
mdm.KeyManagementURL: "https://mdm.example.com:443",
|
||||
mdm.KeyBlockInbound: true,
|
||||
mdm.KeyRosenpassEnabled: true,
|
||||
}))
|
||||
|
||||
s, ctx, profName, username, _ := setupServerWithProfile(t)
|
||||
|
||||
blockInbound := false
|
||||
rosenpassEnabled := false
|
||||
_, err := s.SetConfig(ctx, &proto.SetConfigRequest{
|
||||
@@ -137,12 +161,11 @@ func TestSetConfig_MDMReject_AllOrNothing(t *testing.T) {
|
||||
// enforced field AND a non-enforced field (RosenpassEnabled).
|
||||
// The whole request must be rejected — non-conflicting fields are not
|
||||
// applied either.
|
||||
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
|
||||
s, ctx, profName, username, cfgPath := setupServerWithProfile(t)
|
||||
withMDMPolicy(t, s, mdm.NewPolicy(map[string]any{
|
||||
mdm.KeyManagementURL: "https://mdm.example.com:443",
|
||||
}))
|
||||
|
||||
s, ctx, profName, username, cfgPath := setupServerWithProfile(t)
|
||||
|
||||
rosenpassEnabled := true
|
||||
_, err := s.SetConfig(ctx, &proto.SetConfigRequest{
|
||||
ProfileName: profName,
|
||||
@@ -164,12 +187,11 @@ func TestSetConfig_MDMReject_AllOrNothing(t *testing.T) {
|
||||
func TestSetConfig_MDMAllow_NonManagedFields(t *testing.T) {
|
||||
// MDM enforces ManagementURL but the user only writes RosenpassEnabled.
|
||||
// Request must succeed.
|
||||
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
|
||||
s, ctx, profName, username, _ := setupServerWithProfile(t)
|
||||
withMDMPolicy(t, s, mdm.NewPolicy(map[string]any{
|
||||
mdm.KeyManagementURL: "https://mdm.example.com:443",
|
||||
}))
|
||||
|
||||
s, ctx, profName, username, _ := setupServerWithProfile(t)
|
||||
|
||||
rosenpassEnabled := true
|
||||
resp, err := s.SetConfig(ctx, &proto.SetConfigRequest{
|
||||
ProfileName: profName,
|
||||
@@ -198,12 +220,11 @@ func TestSetConfig_MDMAllow_ManagementURLPortNormalized(t *testing.T) {
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
|
||||
s, ctx, profName, username, _ := setupServerWithProfile(t)
|
||||
withMDMPolicy(t, s, mdm.NewPolicy(map[string]any{
|
||||
mdm.KeyManagementURL: tc.mdmURL,
|
||||
}))
|
||||
|
||||
s, ctx, profName, username, _ := setupServerWithProfile(t)
|
||||
|
||||
rosenpassEnabled := true
|
||||
resp, err := s.SetConfig(ctx, &proto.SetConfigRequest{
|
||||
ProfileName: profName,
|
||||
@@ -220,9 +241,8 @@ func TestSetConfig_MDMAllow_ManagementURLPortNormalized(t *testing.T) {
|
||||
|
||||
func TestSetConfig_MDMEmpty_NoEnforcement(t *testing.T) {
|
||||
// No MDM policy active: any field can be written.
|
||||
withMDMPolicy(t, mdm.NewPolicy(nil))
|
||||
|
||||
s, ctx, profName, username, _ := setupServerWithProfile(t)
|
||||
withMDMPolicy(t, s, mdm.NewPolicy(nil))
|
||||
|
||||
resp, err := s.SetConfig(ctx, &proto.SetConfigRequest{
|
||||
ProfileName: profName,
|
||||
|
||||
@@ -2,6 +2,7 @@ import { type ReactNode } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { Browser } from "@wailsio/runtime";
|
||||
import { DownloadIcon, NotepadText } from "lucide-react";
|
||||
import { Update as UpdateSvc } from "@bindings/services";
|
||||
import { Button } from "@/components/buttons/Button";
|
||||
import { useClientVersion } from "@/contexts/ClientVersionContext";
|
||||
import { cn } from "@/lib/cn";
|
||||
@@ -14,6 +15,12 @@ function openUrl(url: string) {
|
||||
});
|
||||
}
|
||||
|
||||
function openInstallerDownload() {
|
||||
UpdateSvc.DownloadURL()
|
||||
.then(openUrl)
|
||||
.catch(() => openUrl(GITHUB_RELEASES));
|
||||
}
|
||||
|
||||
export function UpdateVersionCard() {
|
||||
const { t } = useTranslation();
|
||||
const { updateVersion, enforced, triggerUpdate } = useClientVersion();
|
||||
@@ -37,11 +44,7 @@ export function UpdateVersionCard() {
|
||||
{t("update.card.installNow")}
|
||||
</Button>
|
||||
) : (
|
||||
<Button
|
||||
variant={"primary"}
|
||||
size={"xs"}
|
||||
onClick={() => openUrl(GITHUB_RELEASES)}
|
||||
>
|
||||
<Button variant={"primary"} size={"xs"} onClick={openInstallerDownload}>
|
||||
<DownloadIcon size={14} />
|
||||
{t("update.card.getInstaller")}
|
||||
</Button>
|
||||
|
||||
@@ -281,6 +281,9 @@ func newApplication(onSecondInstance func()) *application.App {
|
||||
Linux: application.LinuxOptions{
|
||||
ProgramName: "netbird",
|
||||
},
|
||||
Windows: application.WindowsOptions{
|
||||
WndProcInterceptor: endSessionInterceptor(),
|
||||
},
|
||||
SingleInstance: &application.SingleInstanceOptions{
|
||||
UniqueID: "io.netbird.ui",
|
||||
OnSecondInstanceLaunch: func(_ application.SecondInstanceData) {
|
||||
@@ -367,6 +370,9 @@ func newMainWindow(app *application.App, prefStore *preferences.Store) *applicat
|
||||
|
||||
// Hide instead of quit on close; "really quit" is reached via tray -> Quit.
|
||||
window.RegisterHook(events.Common.WindowClosing, func(e *application.WindowEvent) {
|
||||
if services.ShuttingDown() {
|
||||
return
|
||||
}
|
||||
e.Cancel()
|
||||
window.Hide()
|
||||
})
|
||||
|
||||
24
client/ui/services/shutdown.go
Normal file
24
client/ui/services/shutdown.go
Normal file
@@ -0,0 +1,24 @@
|
||||
package services
|
||||
|
||||
import "sync/atomic"
|
||||
|
||||
var (
|
||||
sessionEnding atomic.Bool
|
||||
quitting atomic.Bool
|
||||
)
|
||||
|
||||
func BeginSessionEnd() {
|
||||
sessionEnding.Store(true)
|
||||
}
|
||||
|
||||
func AbortSessionEnd() {
|
||||
sessionEnding.Store(false)
|
||||
}
|
||||
|
||||
func BeginShutdown() {
|
||||
quitting.Store(true)
|
||||
}
|
||||
|
||||
func ShuttingDown() bool {
|
||||
return sessionEnding.Load() || quitting.Load()
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/netbirdio/netbird/client/proto"
|
||||
"github.com/netbirdio/netbird/client/ui/updater"
|
||||
"github.com/netbirdio/netbird/version"
|
||||
)
|
||||
|
||||
// UpdateResult mirrors TriggerUpdateResponse.
|
||||
@@ -33,6 +34,12 @@ func (s *Update) GetState() updater.State {
|
||||
return s.holder.Get()
|
||||
}
|
||||
|
||||
// DownloadURL returns the platform-appropriate installer download link for
|
||||
// manual (non-enforced) updates.
|
||||
func (s *Update) DownloadURL() string {
|
||||
return version.DownloadUrl()
|
||||
}
|
||||
|
||||
// Quit exits the app. Scheduled off the calling goroutine so the JS caller's
|
||||
// response returns before the runtime tears down.
|
||||
func (s *Update) Quit() {
|
||||
|
||||
@@ -154,6 +154,9 @@ func NewWindowManager(app *application.App, mainWindow *application.WebviewWindo
|
||||
})
|
||||
// Hide (not destroy) on close to keep React state; reset to General for a flash-free reopen.
|
||||
s.settings.RegisterHook(events.Common.WindowClosing, func(e *application.WindowEvent) {
|
||||
if ShuttingDown() {
|
||||
return
|
||||
}
|
||||
e.Cancel()
|
||||
s.app.Event.Emit(EventSettingsOpen, "general")
|
||||
s.settings.Hide()
|
||||
@@ -393,6 +396,9 @@ func (s *WindowManager) CloseWelcome() {
|
||||
// OpenError shows the custom error dialog; title/message are pre-localised and ride in the
|
||||
// start URL. A second error replaces the open one via SetURL. Singleton, destroyed on close.
|
||||
func (s *WindowManager) OpenError(title, message string) {
|
||||
if ShuttingDown() {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
startURL := errorDialogURL(title, message)
|
||||
|
||||
7
client/ui/shutdown_other.go
Normal file
7
client/ui/shutdown_other.go
Normal file
@@ -0,0 +1,7 @@
|
||||
//go:build !windows && !android && !ios && !freebsd && !js
|
||||
|
||||
package main
|
||||
|
||||
func endSessionInterceptor() func(hwnd uintptr, msg uint32, wParam, lParam uintptr) (uintptr, bool) {
|
||||
return nil
|
||||
}
|
||||
36
client/ui/shutdown_windows.go
Normal file
36
client/ui/shutdown_windows.go
Normal file
@@ -0,0 +1,36 @@
|
||||
//go:build windows
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/netbirdio/netbird/client/ui/services"
|
||||
)
|
||||
|
||||
const (
|
||||
wmQueryEndSession = 0x0011
|
||||
wmEndSession = 0x0016
|
||||
)
|
||||
|
||||
func endSessionInterceptor() func(hwnd uintptr, msg uint32, wParam, lParam uintptr) (uintptr, bool) {
|
||||
return func(_ uintptr, msg uint32, wParam, _ uintptr) (uintptr, bool) {
|
||||
switch msg {
|
||||
case wmQueryEndSession:
|
||||
services.BeginSessionEnd()
|
||||
return 1, true
|
||||
case wmEndSession:
|
||||
if wParam == 0 {
|
||||
services.AbortSessionEnd()
|
||||
return 0, true
|
||||
}
|
||||
log.Info("windows session is ending; exiting immediately")
|
||||
os.Exit(0)
|
||||
return 0, true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -32,9 +32,8 @@ const (
|
||||
|
||||
quitDownTimeout = 5 * time.Second
|
||||
|
||||
urlGitHubRepo = "https://github.com/netbirdio/netbird"
|
||||
urlGitHubReleases = "https://github.com/netbirdio/netbird/releases/latest"
|
||||
urlDocs = "https://docs.netbird.io"
|
||||
urlGitHubRepo = "https://github.com/netbirdio/netbird"
|
||||
urlDocs = "https://docs.netbird.io"
|
||||
)
|
||||
|
||||
// TrayServices bundles the services the tray menu needs, grouped so NewTray
|
||||
@@ -453,6 +452,7 @@ func (t *Tray) buildMenu() *application.Menu {
|
||||
}
|
||||
|
||||
func (t *Tray) handleQuit() {
|
||||
services.BeginShutdown()
|
||||
t.profileMu.Lock()
|
||||
if t.switchCancel != nil {
|
||||
t.switchCancel()
|
||||
|
||||
@@ -25,6 +25,9 @@ type sendFn func(notifications.NotificationOptions) error
|
||||
// event-dispatch goroutine that panic is fatal process-wide; recover() turns
|
||||
// it into a logged no-op.
|
||||
func safeSendNotification(send sendFn, what string, opts notifications.NotificationOptions) (err error) {
|
||||
if services.ShuttingDown() {
|
||||
return nil
|
||||
}
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Errorf("notify %s: recovered from panic (notification bus unavailable): %v", what, r)
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"github.com/netbirdio/netbird/client/ui/services"
|
||||
"github.com/netbirdio/netbird/client/ui/updater"
|
||||
"github.com/netbirdio/netbird/version"
|
||||
)
|
||||
|
||||
// trayUpdater owns the tray UI that reacts to auto-update. Composed inside Tray.
|
||||
@@ -76,15 +77,15 @@ func (u *trayUpdater) applyLanguage() {
|
||||
u.refreshMenuItem(state)
|
||||
}
|
||||
|
||||
// handleClick opens the GitHub releases page when not Enforced, otherwise shows
|
||||
// the progress page and asks the daemon to start the installer.
|
||||
// handleClick opens the installer download link when not Enforced, otherwise
|
||||
// shows the progress page and asks the daemon to start the installer.
|
||||
func (u *trayUpdater) handleClick() {
|
||||
u.mu.Lock()
|
||||
state := u.state
|
||||
u.mu.Unlock()
|
||||
|
||||
if !state.Enforced {
|
||||
_ = u.app.Browser.OpenURL(urlGitHubReleases)
|
||||
_ = u.app.Browser.OpenURL(version.DownloadUrl())
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -109,7 +109,7 @@ sequenceDiagram
|
||||
Chk->>Inj: continue
|
||||
Inj->>Inj: inject NetBird identity headers per provider config
|
||||
Inj->>Grd: continue
|
||||
Grd->>Grd: enforce model allowlist
|
||||
Grd->>Grd: enforce per-provider allowlist (fail-closed backstop)
|
||||
Grd->>Up: forward (over WireGuard)
|
||||
Up-->>Resp: response (JSON or SSE stream)
|
||||
Resp->>Resp: parse usage tokens, completion
|
||||
@@ -135,6 +135,21 @@ sequenceDiagram
|
||||
(`redact_pii = settings.RedactPii`). Phones, emails, credit cards,
|
||||
PII names — see `redact.go` for the full set. See
|
||||
[`modules/31-proxy-middleware-builtin.md`](modules/31-proxy-middleware-builtin.md).
|
||||
- The model allowlist is enforced in TWO places. `CheckLLMPolicyLimits`
|
||||
is authoritative: it resolves the policy that governs this
|
||||
(provider, caller-groups) and denies (`deny_code = llm_policy.model_blocked`)
|
||||
when no applicable policy permits the model — so an allowlist scoped to
|
||||
one group/provider never leaks to another, and an un-guardrailed policy
|
||||
is genuinely unrestricted. `llm_guardrail` is a per-provider fail-closed
|
||||
backstop: it only carries an allowlist for a provider every authorising
|
||||
policy restricts, and blocks unknown/undetermined models even when
|
||||
management is unreachable. Because that backstop allowlist is the UNION
|
||||
of every restricting policy's models, per-group narrowing lives only in
|
||||
the authoritative check: during a `CheckLLMPolicyLimits` outage
|
||||
`llm_limit_check` fails open, so a caller can reach any model in the
|
||||
provider's union — a group scoped to model A could reach model B if
|
||||
another group restricts the same provider to B. This is the documented
|
||||
fail-open trade-off; a future flag may switch it to fail-closed.
|
||||
- SSE streaming requires special handling on the response side; the
|
||||
parser must handle partial chunks without buffering the whole
|
||||
stream. See [`modules/32-proxy-llm-parsers.md`](modules/32-proxy-llm-parsers.md).
|
||||
|
||||
@@ -122,7 +122,7 @@ At request time the path is independent: the proxy calls `SelectPolicyForRequest
|
||||
| on_request | 1 | `llm_router` | `{"providers":[{id, models[], upstream_*, auth_header_*, allowed_group_ids[]}]}` | **true** |
|
||||
| on_request | 2 | `llm_limit_check` | `{}` | – |
|
||||
| on_request | 3 | `llm_identity_inject` | `{"providers":[{provider_id, header_pair?, json_metadata?, extra_headers?}]}` | **true** |
|
||||
| on_request | 4 | `llm_guardrail` | `{"model_allowlist"?, "prompt_capture":{enabled,redact_pii}}` | – |
|
||||
| on_request | 4 | `llm_guardrail` | `{"provider_allowlists"?: {providerID: []model}, "prompt_capture":{enabled,redact_pii}}` | – |
|
||||
| on_response | 5 | `llm_limit_record` | `{}` (runs LAST at runtime) | – |
|
||||
| on_response | 6 | `cost_meter` | `{}` | – |
|
||||
| on_response | 7 | `llm_response_parser` | `{"capture_completion": <bool>, "redact_pii"?: true}` | – |
|
||||
|
||||
@@ -244,7 +244,7 @@ no mocks. Tests: `TestChain_AllowPath_StampsAttributionAndRecordsCounter`
|
||||
| `llm_router` | `{providers: [{id, models, upstream_scheme, upstream_host, upstream_path?, auth_header_name, auth_header_value, allowed_group_ids}]}` |
|
||||
| `llm_limit_check` | `{}` — pulls `MgmtClient` from `FactoryContext` |
|
||||
| `llm_identity_inject` | `{providers: [{provider_id, header_pair?|json_metadata?, extra_headers?}]}` |
|
||||
| `llm_guardrail` | `{model_allowlist: []string, prompt_capture: {enabled, redact_pii}}` |
|
||||
| `llm_guardrail` | `{provider_allowlists: {providerID: []string}, prompt_capture: {enabled, redact_pii}}` — allowlist keyed by resolved provider id; a provider absent from the map is unrestricted (fail-closed backstop; authoritative per-policy/group check is management's `CheckLLMPolicyLimits`) |
|
||||
| `llm_response_parser` | `{redact_pii?, capture_completion?: *bool}` |
|
||||
| `cost_meter` | `{pricing_path?}` (basename inside data-dir; defaults `pricing.yaml`) |
|
||||
| `llm_limit_record` | `{}` — same pattern as `llm_limit_check` |
|
||||
|
||||
209
e2e/agentnetwork/guardrail_block_test.go
Normal file
209
e2e/agentnetwork/guardrail_block_test.go
Normal file
@@ -0,0 +1,209 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/e2e/harness"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// pathRoutedGuardrailCase is one provider's self-contained scenario: its own
|
||||
// provider, its own guardrail whose allowlist holds ONLY that provider's
|
||||
// allowed model, and its own policy. Each case runs in isolation (its own
|
||||
// proxy + client), so the guardrail the proxy enforces contains exactly this
|
||||
// provider's model — never a mixed cross-provider list.
|
||||
type pathRoutedGuardrailCase struct {
|
||||
name string
|
||||
catalogID string // agent-network catalog provider id
|
||||
wire string // harness.WireVertex | harness.WireBedrock
|
||||
allowEntry string // the single model id put on the guardrail allowlist
|
||||
allowModel string // model id sent that MUST be served (200)
|
||||
blockModel string // model id sent that MUST be denied (403 model_blocked)
|
||||
}
|
||||
|
||||
// TestGuardrailBlocksUnselectedModel_PathRouted is the end-to-end regression
|
||||
// guard for the customer report that a model-allowlist guardrail attached to a
|
||||
// policy has no effect for PATH-ROUTED providers — where the model travels in
|
||||
// the URL, not the JSON body: Google Vertex (…/models/{model}:rawPredict) and
|
||||
// AWS Bedrock (/model/{id}/invoke).
|
||||
//
|
||||
// Each provider is tested in isolation with a guardrail allowlisting a single
|
||||
// model of its own: the allowed model (in the URL path) is served (200) and an
|
||||
// unselected model (in the URL path) is denied 403 by the guardrail
|
||||
// (llm_policy.model_blocked) before the upstream. The Vertex case mirrors the
|
||||
// customer verbatim — allow Sonnet, and the unselected model is the exact
|
||||
// claude-opus-4-6 they reported reaching the model unblocked. The Bedrock case
|
||||
// sends a region-prefixed, versioned inference-profile id so URL-path model
|
||||
// normalization is exercised too.
|
||||
//
|
||||
// The provider is catch-all (no models), so the router forwards any model and a
|
||||
// 403 can only come from the guardrail, never model_not_routable. Only the
|
||||
// upstream LLM is mocked (the vLLM nginx answers any path with 200); management
|
||||
// synth/reconcile, the proxy middleware chain (URL-path model extraction,
|
||||
// router, guardrail) and the tunnel are all real, and the guardrail denies
|
||||
// before the upstream is dialed so the mock cannot influence the block. A
|
||||
// static bearer api key is used so the router injects a static Authorization
|
||||
// header instead of minting a GCP token — the only reason path-routed providers
|
||||
// normally need live credentials — so the test runs with none and is always on.
|
||||
func TestGuardrailBlocksUnselectedModel_PathRouted(t *testing.T) {
|
||||
cases := []pathRoutedGuardrailCase{
|
||||
{
|
||||
name: "vertex",
|
||||
catalogID: "vertex_ai_api",
|
||||
wire: harness.WireVertex,
|
||||
allowEntry: "claude-sonnet-4-5",
|
||||
allowModel: "claude-sonnet-4-5",
|
||||
blockModel: "claude-opus-4-6", // the customer-reported model
|
||||
},
|
||||
{
|
||||
name: "bedrock",
|
||||
catalogID: "bedrock_api",
|
||||
wire: harness.WireBedrock,
|
||||
allowEntry: "anthropic.claude-sonnet-4-5", // normalized catalog id
|
||||
allowModel: "us.anthropic.claude-sonnet-4-5-v1:0",
|
||||
blockModel: "us.anthropic.claude-opus-4-8-v1:0",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
tc := tc
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
runPathRoutedGuardrailCase(t, tc)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func runPathRoutedGuardrailCase(t *testing.T, tc pathRoutedGuardrailCase) {
|
||||
t.Helper()
|
||||
|
||||
const (
|
||||
vertexProject = "e2e-project"
|
||||
vertexRegion = "global"
|
||||
)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
vllm, err := harness.StartVLLM(ctx, srv)
|
||||
require.NoError(t, err, "start mock upstream")
|
||||
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
|
||||
|
||||
grp, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-guardrail-" + tc.name})
|
||||
require.NoError(t, err, "create group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grp.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-guardrail-" + tc.name + "-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grp.Id},
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext")
|
||||
|
||||
// Catch-all provider (no models) so the router forwards any model; a static
|
||||
// bearer key means the router injects a static auth header instead of minting
|
||||
// a GCP token. Bootstraps the cluster if it isn't already.
|
||||
staticKey := "static-e2e-token"
|
||||
prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: tc.name,
|
||||
ProviderId: tc.catalogID,
|
||||
UpstreamUrl: vllm.URL,
|
||||
ApiKey: &staticKey,
|
||||
Enabled: ptr(true),
|
||||
BootstrapCluster: ptr(harness.AgentNetworkCluster),
|
||||
})
|
||||
require.NoError(t, err, "create %s provider", tc.name)
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
|
||||
|
||||
// Guardrail allowlisting ONLY this provider's allowed model.
|
||||
var gr api.AgentNetworkGuardrailRequest
|
||||
gr.Name = "e2e-guardrail-" + tc.name
|
||||
gr.Checks.ModelAllowlist.Enabled = true
|
||||
gr.Checks.ModelAllowlist.Models = []string{tc.allowEntry}
|
||||
guard, err := srv.CreateGuardrail(ctx, gr)
|
||||
require.NoError(t, err, "create guardrail")
|
||||
t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), guard.Id) })
|
||||
|
||||
enabled := true
|
||||
pol, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-guardrail-" + tc.name,
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grp.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
GuardrailIds: &[]string{guard.Id},
|
||||
})
|
||||
require.NoError(t, err, "create policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) })
|
||||
|
||||
settings, err := srv.GetSettings(ctx)
|
||||
require.NoError(t, err, "read settings")
|
||||
require.NotEmpty(t, settings.Endpoint, "endpoint must be assigned")
|
||||
|
||||
proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-guardrail-"+tc.name+"-proxy")
|
||||
require.NoError(t, err, "mint proxy token")
|
||||
px, err := harness.StartProxy(ctx, srv, proxyToken)
|
||||
require.NoError(t, err, "start proxy")
|
||||
t.Cleanup(func() { _ = px.Terminate(context.Background()) })
|
||||
|
||||
cl, err := harness.StartClient(ctx, srv, sk.Key)
|
||||
require.NoError(t, err, "start client")
|
||||
t.Cleanup(func() { _ = cl.Terminate(context.Background()) })
|
||||
|
||||
require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management")
|
||||
// Probe first: the GET resolves the endpoint and its first packet wakes the
|
||||
// lazy proxy peer, so WaitProxyPeer then observes it connected.
|
||||
proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint)
|
||||
require.NoError(t, err, "resolve endpoint to proxy IP")
|
||||
if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil {
|
||||
t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background()))
|
||||
}
|
||||
|
||||
send := func(model string) (int, string) {
|
||||
var code int
|
||||
var body string
|
||||
var cerr error
|
||||
switch tc.wire {
|
||||
case harness.WireVertex:
|
||||
code, body, cerr = cl.Vertex(ctx, settings.Endpoint, proxyIP, vertexProject, vertexRegion, model, "Reply with exactly: pong", "")
|
||||
case harness.WireBedrock:
|
||||
code, body, cerr = cl.Bedrock(ctx, settings.Endpoint, proxyIP, model, "Reply with exactly: pong", "")
|
||||
default:
|
||||
t.Fatalf("unsupported wire %q", tc.wire)
|
||||
}
|
||||
require.NoError(t, cerr, "request must reach the proxy for %s", tc.name)
|
||||
return code, body
|
||||
}
|
||||
|
||||
// Allowed model (in the URL path) is served. Retry to absorb tunnel/DNS
|
||||
// jitter on the first call over the freshly warmed tunnel.
|
||||
var code int
|
||||
var body string
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
code, body = send(tc.allowModel)
|
||||
if code == 200 {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
assert.Equal(t, 200, code,
|
||||
"allowed %s model (URL path) must be served; body: %s\n=== proxy logs ===\n%s", tc.name, body, px.Logs(context.Background()))
|
||||
|
||||
// Unselected model (in the URL path) must be blocked by the guardrail.
|
||||
code, body = send(tc.blockModel)
|
||||
assert.Equal(t, 403, code,
|
||||
"unselected %s model (URL path) must be denied, not served; body: %s\n=== proxy logs ===\n%s", tc.name, body, px.Logs(context.Background()))
|
||||
assert.Contains(t, body, "llm_policy.model_blocked",
|
||||
"%s denial must come from the guardrail allowlist, not routing; body: %s", tc.name, body)
|
||||
}
|
||||
205
e2e/agentnetwork/guardrail_groupswitch_test.go
Normal file
205
e2e/agentnetwork/guardrail_groupswitch_test.go
Normal file
@@ -0,0 +1,205 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/e2e/harness"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// TestGuardrailGroupSwitchTakesEffectAfterTTL proves that moving a peer between
|
||||
// groups flips its model-allowlist decision once the proxy's tunnel-peer cache
|
||||
// expires. The peer's groups reach the guardrail via ValidateTunnelPeer, which
|
||||
// the proxy caches; the switch is invisible until that cache expires. The proxy
|
||||
// runs with a short NB_PROXY_TUNNEL_CACHE_TTL so the flip happens in seconds
|
||||
// instead of the 5-minute default.
|
||||
//
|
||||
// Setup: one catch-all provider declaring modelA + modelB; polA (grpA -> allow
|
||||
// modelA) and polB (grpB -> allow modelB). The client starts in grpA. modelA is
|
||||
// served and modelB denied; after switching the client grpA -> grpB, modelB is
|
||||
// served and modelA denied. The cross-group deny comes from management's
|
||||
// per-policy/group CheckLLMPolicyLimits (the proxy backstop carries the union).
|
||||
func TestGuardrailGroupSwitchTakesEffectAfterTTL(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
const (
|
||||
modelA = "e2e-model-a"
|
||||
modelB = "e2e-model-b"
|
||||
// Short tunnel-cache TTL so a group switch propagates in seconds.
|
||||
// Exercises the NB_PROXY_TUNNEL_CACHE_TTL override.
|
||||
cacheTTL = 3 * time.Second
|
||||
)
|
||||
|
||||
vllm, err := harness.StartVLLM(ctx, srv)
|
||||
require.NoError(t, err, "start mock upstream")
|
||||
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
|
||||
|
||||
grpA, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-gswitch-a"})
|
||||
require.NoError(t, err, "create group A")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grpA.Id) })
|
||||
|
||||
grpB, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-gswitch-b"})
|
||||
require.NoError(t, err, "create group B")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grpB.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-gswitch-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grpA.Id}, // client starts in group A
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext")
|
||||
|
||||
staticKey := "static-e2e-token"
|
||||
prov, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: "gswitch",
|
||||
ProviderId: "openai_api",
|
||||
UpstreamUrl: vllm.URL,
|
||||
ApiKey: &staticKey,
|
||||
Enabled: ptr(true),
|
||||
Models: &[]api.AgentNetworkProviderModel{
|
||||
{Id: modelA, InputPer1k: 0.001, OutputPer1k: 0.001},
|
||||
{Id: modelB, InputPer1k: 0.001, OutputPer1k: 0.001},
|
||||
},
|
||||
BootstrapCluster: ptr(harness.AgentNetworkCluster),
|
||||
})
|
||||
require.NoError(t, err, "create provider")
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
|
||||
|
||||
mkGuard := func(name, model string) api.AgentNetworkGuardrail {
|
||||
var gr api.AgentNetworkGuardrailRequest
|
||||
gr.Name = name
|
||||
gr.Checks.ModelAllowlist.Enabled = true
|
||||
gr.Checks.ModelAllowlist.Models = []string{model}
|
||||
g, gerr := srv.CreateGuardrail(ctx, gr)
|
||||
require.NoError(t, gerr, "create guardrail %s", name)
|
||||
t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), g.Id) })
|
||||
return g
|
||||
}
|
||||
gA := mkGuard("e2e-gswitch-a", modelA)
|
||||
gB := mkGuard("e2e-gswitch-b", modelB)
|
||||
|
||||
enabled := true
|
||||
polA, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-gswitch-a",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grpA.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
GuardrailIds: &[]string{gA.Id},
|
||||
})
|
||||
require.NoError(t, err, "create policy A")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), polA.Id) })
|
||||
|
||||
polB, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-gswitch-b",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grpB.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
GuardrailIds: &[]string{gB.Id},
|
||||
})
|
||||
require.NoError(t, err, "create policy B")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), polB.Id) })
|
||||
|
||||
settings, err := srv.GetSettings(ctx)
|
||||
require.NoError(t, err, "read settings")
|
||||
require.NotEmpty(t, settings.Endpoint, "endpoint must be assigned")
|
||||
|
||||
proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-gswitch-proxy")
|
||||
require.NoError(t, err, "mint proxy token")
|
||||
px, err := harness.StartProxy(ctx, srv, proxyToken, map[string]string{
|
||||
"NB_PROXY_TUNNEL_CACHE_TTL": cacheTTL.String(),
|
||||
})
|
||||
require.NoError(t, err, "start proxy")
|
||||
t.Cleanup(func() { _ = px.Terminate(context.Background()) })
|
||||
|
||||
cl, err := harness.StartClient(ctx, srv, sk.Key)
|
||||
require.NoError(t, err, "start client")
|
||||
t.Cleanup(func() { _ = cl.Terminate(context.Background()) })
|
||||
|
||||
require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management")
|
||||
proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint)
|
||||
require.NoError(t, err, "resolve endpoint to proxy IP")
|
||||
if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil {
|
||||
t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background()))
|
||||
}
|
||||
|
||||
send := func(model string) (int, string) {
|
||||
code, body, cerr := cl.Chat(ctx, settings.Endpoint, proxyIP, harness.WireChat, model, "Reply with exactly: pong", "")
|
||||
require.NoError(t, cerr, "request must reach the proxy")
|
||||
return code, body
|
||||
}
|
||||
sendUntil := func(model string, want int, timeout time.Duration) (int, string) {
|
||||
var code int
|
||||
var body string
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
code, body = send(model)
|
||||
if code == want {
|
||||
return code, body
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
return code, body
|
||||
}
|
||||
|
||||
// Phase 1 — client is in group A: modelA served, modelB denied.
|
||||
code, body := sendUntil(modelA, 200, 90*time.Second)
|
||||
assert.Equal(t, 200, code,
|
||||
"group-A model must be served while the client is in group A; body: %s\n=== proxy logs ===\n%s", body, px.Logs(context.Background()))
|
||||
code, body = send(modelB)
|
||||
assert.Equal(t, 403, code,
|
||||
"group-B model must be denied while the client is in group A; body: %s", body)
|
||||
assert.Contains(t, body, "llm_policy.model_blocked")
|
||||
|
||||
// Switch the client peer from group A to group B.
|
||||
peerID := clientPeerInGroup(t, ctx, grpA.Id)
|
||||
_, err = srv.API().Groups.Update(ctx, grpB.Id, api.PutApiGroupsGroupIdJSONRequestBody{
|
||||
Name: grpB.Name,
|
||||
Peers: &[]string{peerID},
|
||||
})
|
||||
require.NoError(t, err, "add peer to group B")
|
||||
_, err = srv.API().Groups.Update(ctx, grpA.Id, api.PutApiGroupsGroupIdJSONRequestBody{
|
||||
Name: grpA.Name,
|
||||
Peers: &[]string{},
|
||||
})
|
||||
require.NoError(t, err, "remove peer from group A")
|
||||
|
||||
// Phase 2 — after the short TTL expires the proxy re-validates the peer,
|
||||
// sees group B, and the decision flips. Poll to absorb TTL + re-validation.
|
||||
code, body = sendUntil(modelB, 200, 60*time.Second)
|
||||
assert.Equal(t, 200, code,
|
||||
"after the group switch + TTL, the group-B model must be served; body: %s\n=== proxy logs ===\n%s", body, px.Logs(context.Background()))
|
||||
code, body = send(modelA)
|
||||
assert.Equal(t, 403, code,
|
||||
"after the switch, the old group-A model must be denied; body: %s", body)
|
||||
assert.Contains(t, body, "llm_policy.model_blocked")
|
||||
}
|
||||
|
||||
// clientPeerInGroup returns the id of the single peer that is a member of the
|
||||
// given group — the test client. The proxy peer is never added to test groups.
|
||||
func clientPeerInGroup(t *testing.T, ctx context.Context, groupID string) string {
|
||||
t.Helper()
|
||||
peers, err := srv.API().Peers.List(ctx)
|
||||
require.NoError(t, err, "list peers")
|
||||
for _, p := range peers {
|
||||
for _, g := range p.Groups {
|
||||
if g.Id == groupID {
|
||||
return p.Id
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("no peer found in group %s", groupID)
|
||||
return ""
|
||||
}
|
||||
201
e2e/agentnetwork/guardrail_multipolicy_test.go
Normal file
201
e2e/agentnetwork/guardrail_multipolicy_test.go
Normal file
@@ -0,0 +1,201 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/e2e/harness"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// TestGuardrailMultiPolicyModelAllowlist: modelSelected served (200), grpOther's
|
||||
// modelOther denied for the grpMain client (403 model_blocked, no cross-group
|
||||
// leak), and openModel on the un-guardrailed policy's provider served (200).
|
||||
func TestGuardrailMultiPolicyModelAllowlist(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
const (
|
||||
modelSelected = "e2e-selected"
|
||||
modelOther = "e2e-other"
|
||||
openModel = "e2e-open"
|
||||
)
|
||||
|
||||
vllm, err := harness.StartVLLM(ctx, srv)
|
||||
require.NoError(t, err, "start mock upstream")
|
||||
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
|
||||
|
||||
grpMain, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-guardrail-mp-main"})
|
||||
require.NoError(t, err, "create main group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grpMain.Id) })
|
||||
|
||||
grpOther, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-guardrail-mp-other"})
|
||||
require.NoError(t, err, "create other group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grpOther.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-guardrail-mp-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grpMain.Id}, // client joins grpMain only
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext")
|
||||
|
||||
staticKey := "static-e2e-token"
|
||||
models := func(ids ...string) *[]api.AgentNetworkProviderModel {
|
||||
out := make([]api.AgentNetworkProviderModel, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
out = append(out, api.AgentNetworkProviderModel{Id: id, InputPer1k: 0.001, OutputPer1k: 0.001})
|
||||
}
|
||||
return &out
|
||||
}
|
||||
|
||||
// pRestricted declares the two guardrailed models so routing is deterministic
|
||||
// (model -> provider). Created first, so it carries the bootstrap cluster.
|
||||
pRestricted, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: "restricted",
|
||||
ProviderId: "openai_api",
|
||||
UpstreamUrl: vllm.URL,
|
||||
ApiKey: &staticKey,
|
||||
Enabled: ptr(true),
|
||||
Models: models(modelSelected, modelOther),
|
||||
BootstrapCluster: ptr(harness.AgentNetworkCluster),
|
||||
})
|
||||
require.NoError(t, err, "create restricted provider")
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), pRestricted.Id) })
|
||||
|
||||
pOpen, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: "open",
|
||||
ProviderId: "openai_api",
|
||||
UpstreamUrl: vllm.URL,
|
||||
ApiKey: &staticKey,
|
||||
Enabled: ptr(true),
|
||||
Models: models(openModel),
|
||||
})
|
||||
require.NoError(t, err, "create open provider")
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), pOpen.Id) })
|
||||
|
||||
mkGuardrail := func(name, model string) api.AgentNetworkGuardrail {
|
||||
var gr api.AgentNetworkGuardrailRequest
|
||||
gr.Name = name
|
||||
gr.Checks.ModelAllowlist.Enabled = true
|
||||
gr.Checks.ModelAllowlist.Models = []string{model}
|
||||
g, gerr := srv.CreateGuardrail(ctx, gr)
|
||||
require.NoError(t, gerr, "create guardrail %s", name)
|
||||
t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), g.Id) })
|
||||
return g
|
||||
}
|
||||
gMain := mkGuardrail("e2e-guardrail-mp-main", modelSelected)
|
||||
gOther := mkGuardrail("e2e-guardrail-mp-other", modelOther)
|
||||
|
||||
enabled := true
|
||||
// polMain: grpMain restricted to modelSelected on pRestricted.
|
||||
polMain, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-guardrail-mp-main",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grpMain.Id},
|
||||
DestinationProviderIds: []string{pRestricted.Id},
|
||||
GuardrailIds: &[]string{gMain.Id},
|
||||
})
|
||||
require.NoError(t, err, "create main policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), polMain.Id) })
|
||||
|
||||
// polOther: grpOther restricted to modelOther on the SAME provider. The
|
||||
// client is not in grpOther, so modelOther must never be usable by it.
|
||||
polOther, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-guardrail-mp-other",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grpOther.Id},
|
||||
DestinationProviderIds: []string{pRestricted.Id},
|
||||
GuardrailIds: &[]string{gOther.Id},
|
||||
})
|
||||
require.NoError(t, err, "create other policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), polOther.Id) })
|
||||
|
||||
// polOpen: grpMain on pOpen with NO guardrail — unrestricted.
|
||||
polOpen, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-guardrail-mp-open",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grpMain.Id},
|
||||
DestinationProviderIds: []string{pOpen.Id},
|
||||
})
|
||||
require.NoError(t, err, "create open policy")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), polOpen.Id) })
|
||||
|
||||
settings, err := srv.GetSettings(ctx)
|
||||
require.NoError(t, err, "read settings")
|
||||
require.NotEmpty(t, settings.Endpoint, "endpoint must be assigned")
|
||||
|
||||
proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-guardrail-mp-proxy")
|
||||
require.NoError(t, err, "mint proxy token")
|
||||
px, err := harness.StartProxy(ctx, srv, proxyToken)
|
||||
require.NoError(t, err, "start proxy")
|
||||
t.Cleanup(func() { _ = px.Terminate(context.Background()) })
|
||||
|
||||
cl, err := harness.StartClient(ctx, srv, sk.Key)
|
||||
require.NoError(t, err, "start client")
|
||||
t.Cleanup(func() { _ = cl.Terminate(context.Background()) })
|
||||
|
||||
require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management")
|
||||
proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint)
|
||||
require.NoError(t, err, "resolve endpoint to proxy IP")
|
||||
if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil {
|
||||
t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background()))
|
||||
}
|
||||
|
||||
send := func(model string) (int, string) {
|
||||
code, body, cerr := cl.Chat(ctx, settings.Endpoint, proxyIP, harness.WireChat, model, "Reply with exactly: pong", "")
|
||||
require.NoError(t, cerr, "request must reach the proxy")
|
||||
return code, body
|
||||
}
|
||||
// sendUntil200 absorbs first-call tunnel/DNS jitter on the freshly warmed tunnel.
|
||||
sendUntil200 := func(model string) (int, string) {
|
||||
var code int
|
||||
var body string
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
code, body = send(model)
|
||||
if code == 200 {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
return code, body
|
||||
}
|
||||
|
||||
t.Run("selected model allowed for its group", func(t *testing.T) {
|
||||
code, body := sendUntil200(modelSelected)
|
||||
assert.Equal(t, 200, code,
|
||||
"grpMain's allowlisted model must be served; body: %s\n=== proxy logs ===\n%s", body, px.Logs(context.Background()))
|
||||
})
|
||||
|
||||
t.Run("other group's model does not leak", func(t *testing.T) {
|
||||
// modelOther is allowlisted only for grpOther. The grpMain client must be
|
||||
// denied by management's per-policy/group check — not waved through by an
|
||||
// account-wide union. This is the security-critical wrong-ALLOW guard.
|
||||
code, body := send(modelOther)
|
||||
assert.Equal(t, 403, code,
|
||||
"another group's allowlisted model must be denied for this caller; body: %s\n=== proxy logs ===\n%s", body, px.Logs(context.Background()))
|
||||
assert.Contains(t, body, "llm_policy.model_blocked",
|
||||
"denial must be a model-allowlist decision; body: %s", body)
|
||||
})
|
||||
|
||||
t.Run("unguarded policy leaves its provider unrestricted", func(t *testing.T) {
|
||||
// polOpen carries no guardrail, so pOpen is unrestricted for grpMain. The
|
||||
// old account-wide union would have blocked openModel (it is on no
|
||||
// allowlist); it must now be served — the false-DENY guard.
|
||||
code, body := sendUntil200(openModel)
|
||||
assert.Equal(t, 200, code,
|
||||
"an un-guardrailed policy's provider must not be blocked by another policy's allowlist; body: %s\n=== proxy logs ===\n%s", body, px.Logs(context.Background()))
|
||||
})
|
||||
}
|
||||
422
e2e/agentnetwork/guardrail_pergroup_providers_test.go
Normal file
422
e2e/agentnetwork/guardrail_pergroup_providers_test.go
Normal file
@@ -0,0 +1,422 @@
|
||||
//go:build e2e
|
||||
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/e2e/harness"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
// pergroupCase describes one provider surface for the per-group allowlist matrix.
|
||||
// selectedReq/otherReq are the model identifiers as they travel in the request
|
||||
// (URL path for Bedrock/Vertex, body "model" for chat/messages). selectedAllow/
|
||||
// otherAllow are the (normalized) forms the guardrail allowlist holds — for
|
||||
// Bedrock these differ from the request form so path normalization is exercised.
|
||||
type pergroupCase struct {
|
||||
name string
|
||||
catalogID string
|
||||
wire string // "chat", "messages", "vertex", "bedrock"
|
||||
models *[]api.AgentNetworkProviderModel
|
||||
selectedReq string
|
||||
selectedAllow string
|
||||
otherReq string
|
||||
otherAllow string
|
||||
|
||||
providerID string // filled during setup
|
||||
}
|
||||
|
||||
// TestGuardrailPerGroupAllowlist_AllProviders proves the per-policy/group model
|
||||
// allowlist end to end across every always-on provider surface, including the
|
||||
// path-routed ones (Vertex, Bedrock) where the model travels in the URL.
|
||||
//
|
||||
// For each provider two policies target it: grpMain (the client) is allowed only
|
||||
// selectedReq; grpOther (which the client is NOT in) is allowed only otherReq.
|
||||
// The client must get selectedReq served (200) and otherReq denied (403,
|
||||
// llm_policy.model_blocked) — the cross-group no-leak property. The deny is the
|
||||
// authoritative per-policy/group decision from management (the proxy per-provider
|
||||
// backstop carries the union of both models), so this also confirms management
|
||||
// receives the correct normalized model for path-routed providers.
|
||||
func TestGuardrailPerGroupAllowlist_AllProviders(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
const (
|
||||
vertexProject = "e2e-project"
|
||||
vertexRegion = "global"
|
||||
)
|
||||
|
||||
priced := func(ids ...string) *[]api.AgentNetworkProviderModel {
|
||||
out := make([]api.AgentNetworkProviderModel, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
out = append(out, api.AgentNetworkProviderModel{Id: id, InputPer1k: 0.001, OutputPer1k: 0.001})
|
||||
}
|
||||
return &out
|
||||
}
|
||||
|
||||
cases := []*pergroupCase{
|
||||
{
|
||||
name: "openai", catalogID: "openai_api", wire: harness.WireChat,
|
||||
models: priced("oai-model-a", "oai-model-b"),
|
||||
selectedReq: "oai-model-a", selectedAllow: "oai-model-a",
|
||||
otherReq: "oai-model-b", otherAllow: "oai-model-b",
|
||||
},
|
||||
{
|
||||
name: "anthropic", catalogID: "anthropic_api", wire: harness.WireMessages,
|
||||
models: priced("ant-model-a", "ant-model-b"),
|
||||
selectedReq: "ant-model-a", selectedAllow: "ant-model-a",
|
||||
otherReq: "ant-model-b", otherAllow: "ant-model-b",
|
||||
},
|
||||
{
|
||||
// Vertex catalog ids travel bare in the rawPredict path.
|
||||
name: "vertex", catalogID: "vertex_ai_api", wire: "vertex",
|
||||
selectedReq: "claude-sonnet-4-5", selectedAllow: "claude-sonnet-4-5",
|
||||
otherReq: "claude-opus-4-6", otherAllow: "claude-opus-4-6",
|
||||
},
|
||||
{
|
||||
// Bedrock request ids are region-prefixed/versioned; the parser
|
||||
// normalizes them to the catalog key the allowlist holds.
|
||||
name: "bedrock", catalogID: "bedrock_api", wire: "bedrock",
|
||||
selectedReq: "us.anthropic.claude-sonnet-4-5-v1:0", selectedAllow: "anthropic.claude-sonnet-4-5",
|
||||
otherReq: "us.anthropic.claude-opus-4-8-v1:0", otherAllow: "anthropic.claude-opus-4-8",
|
||||
},
|
||||
}
|
||||
|
||||
vllm, err := harness.StartVLLM(ctx, srv)
|
||||
require.NoError(t, err, "start mock upstream")
|
||||
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
|
||||
|
||||
grpMain, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-pergroup-main"})
|
||||
require.NoError(t, err, "create main group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grpMain.Id) })
|
||||
|
||||
grpOther, err := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: "e2e-pergroup-other"})
|
||||
require.NoError(t, err, "create other group")
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), grpOther.Id) })
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-pergroup-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{grpMain.Id},
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext")
|
||||
|
||||
staticKey := "static-e2e-token"
|
||||
enabled := true
|
||||
|
||||
for i, c := range cases {
|
||||
req := api.AgentNetworkProviderRequest{
|
||||
Name: "e2e-pergroup-" + c.name,
|
||||
ProviderId: c.catalogID,
|
||||
UpstreamUrl: vllm.URL,
|
||||
ApiKey: &staticKey,
|
||||
Enabled: ptr(true),
|
||||
Models: c.models,
|
||||
}
|
||||
if i == 0 {
|
||||
req.BootstrapCluster = ptr(harness.AgentNetworkCluster)
|
||||
}
|
||||
prov, perr := srv.CreateProvider(ctx, req)
|
||||
require.NoError(t, perr, "create provider %s", c.name)
|
||||
c.providerID = prov.Id
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) })
|
||||
|
||||
gSel := mkAllowGuardrail(t, ctx, "e2e-pergroup-"+c.name+"-sel", c.selectedAllow)
|
||||
gOth := mkAllowGuardrail(t, ctx, "e2e-pergroup-"+c.name+"-oth", c.otherAllow)
|
||||
|
||||
polMain, merr := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-pergroup-" + c.name + "-main",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grpMain.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
GuardrailIds: &[]string{gSel.Id},
|
||||
})
|
||||
require.NoError(t, merr, "create main policy %s", c.name)
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), polMain.Id) })
|
||||
|
||||
polOther, oerr := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-pergroup-" + c.name + "-other",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{grpOther.Id},
|
||||
DestinationProviderIds: []string{prov.Id},
|
||||
GuardrailIds: &[]string{gOth.Id},
|
||||
})
|
||||
require.NoError(t, oerr, "create other policy %s", c.name)
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), polOther.Id) })
|
||||
}
|
||||
|
||||
settings, err := srv.GetSettings(ctx)
|
||||
require.NoError(t, err, "read settings")
|
||||
require.NotEmpty(t, settings.Endpoint, "endpoint must be assigned")
|
||||
|
||||
proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-pergroup-proxy")
|
||||
require.NoError(t, err, "mint proxy token")
|
||||
px, err := harness.StartProxy(ctx, srv, proxyToken)
|
||||
require.NoError(t, err, "start proxy")
|
||||
t.Cleanup(func() { _ = px.Terminate(context.Background()) })
|
||||
|
||||
cl, err := harness.StartClient(ctx, srv, sk.Key)
|
||||
require.NoError(t, err, "start client")
|
||||
t.Cleanup(func() { _ = cl.Terminate(context.Background()) })
|
||||
|
||||
require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management")
|
||||
proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint)
|
||||
require.NoError(t, err, "resolve endpoint to proxy IP")
|
||||
if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil {
|
||||
t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background()))
|
||||
}
|
||||
|
||||
send := func(c *pergroupCase, model string) (int, string) {
|
||||
var code int
|
||||
var body string
|
||||
var cerr error
|
||||
switch c.wire {
|
||||
case "vertex":
|
||||
code, body, cerr = cl.Vertex(ctx, settings.Endpoint, proxyIP, vertexProject, vertexRegion, model, "Reply with exactly: pong", "")
|
||||
case "bedrock":
|
||||
code, body, cerr = cl.Bedrock(ctx, settings.Endpoint, proxyIP, model, "Reply with exactly: pong", "")
|
||||
default:
|
||||
code, body, cerr = cl.Chat(ctx, settings.Endpoint, proxyIP, c.wire, model, "Reply with exactly: pong", "")
|
||||
}
|
||||
require.NoError(t, cerr, "request must reach the proxy for %s", c.name)
|
||||
return code, body
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
// grpMain's own model is served. Retry to absorb tunnel/DNS jitter on
|
||||
// the first call over the freshly warmed tunnel.
|
||||
var code int
|
||||
var body string
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
code, body = send(c, c.selectedReq)
|
||||
if code == 200 {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
assert.Equal(t, 200, code,
|
||||
"%s: grpMain's allowlisted model must be served; body: %s\n=== proxy logs ===\n%s", c.name, body, px.Logs(context.Background()))
|
||||
|
||||
// grpOther's model must NOT leak to the grpMain client.
|
||||
code, body = send(c, c.otherReq)
|
||||
assert.Equal(t, 403, code,
|
||||
"%s: another group's allowlisted model must be denied for this caller; body: %s\n=== proxy logs ===\n%s", c.name, body, px.Logs(context.Background()))
|
||||
assert.Contains(t, body, "llm_policy.model_blocked",
|
||||
"%s: denial must be a model-allowlist decision, not routing; body: %s", c.name, body)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestGuardrailMultiGroupUser proves the per-policy/group decision for a caller
|
||||
// that belongs to MULTIPLE groups at once. Two scenarios, one shared stack:
|
||||
//
|
||||
// - union across the user's groups: the client is in gUX and gUY, each with
|
||||
// its own policy+guardrail on provider P1 (gUX->union-a, gUY->union-b). The
|
||||
// client may use BOTH models (the union of its groups' allowlists) while a
|
||||
// third, un-allowlisted model is denied.
|
||||
// - an un-guardrailed group lifts the restriction: the client is in gMP and
|
||||
// gMQ on provider P2, where gMP restricts to mix-a but gMQ's policy carries
|
||||
// NO guardrail. Because one applicable policy is unrestricted, the client may
|
||||
// use a model on no allowlist (mix-z) as well as mix-a.
|
||||
func TestGuardrailMultiGroupUser(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
const (
|
||||
unionA = "mg-union-a"
|
||||
unionB = "mg-union-b"
|
||||
unionC = "mg-union-c" // allowlisted by neither group
|
||||
mixA = "mg-mix-a"
|
||||
mixZ = "mg-mix-z" // on no allowlist; reachable only via the un-guardrailed policy
|
||||
)
|
||||
|
||||
priced := func(ids ...string) *[]api.AgentNetworkProviderModel {
|
||||
out := make([]api.AgentNetworkProviderModel, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
out = append(out, api.AgentNetworkProviderModel{Id: id, InputPer1k: 0.001, OutputPer1k: 0.001})
|
||||
}
|
||||
return &out
|
||||
}
|
||||
|
||||
vllm, err := harness.StartVLLM(ctx, srv)
|
||||
require.NoError(t, err, "start mock upstream")
|
||||
t.Cleanup(func() { _ = vllm.Terminate(context.Background()) })
|
||||
|
||||
mkGroup := func(name string) *api.Group {
|
||||
g, gerr := srv.API().Groups.Create(ctx, api.PostApiGroupsJSONRequestBody{Name: name})
|
||||
require.NoError(t, gerr, "create group %s", name)
|
||||
t.Cleanup(func() { _ = srv.API().Groups.Delete(context.Background(), g.Id) })
|
||||
return g
|
||||
}
|
||||
gUX := mkGroup("e2e-mg-union-x")
|
||||
gUY := mkGroup("e2e-mg-union-y")
|
||||
gMP := mkGroup("e2e-mg-mix-p")
|
||||
gMQ := mkGroup("e2e-mg-mix-q")
|
||||
|
||||
ephemeral := false
|
||||
sk, err := srv.API().SetupKeys.Create(ctx, api.PostApiSetupKeysJSONRequestBody{
|
||||
Name: "e2e-mg-client",
|
||||
Type: "reusable",
|
||||
ExpiresIn: 86400,
|
||||
UsageLimit: 0,
|
||||
AutoGroups: []string{gUX.Id, gUY.Id, gMP.Id, gMQ.Id}, // client in all four groups
|
||||
Ephemeral: &ephemeral,
|
||||
})
|
||||
require.NoError(t, err, "mint setup key")
|
||||
require.NotEmpty(t, sk.Key, "setup key plaintext")
|
||||
|
||||
staticKey := "static-e2e-token"
|
||||
enabled := true
|
||||
|
||||
// P1 — union scenario: two restricting policies, one per group.
|
||||
p1, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: "e2e-mg-union",
|
||||
ProviderId: "openai_api",
|
||||
UpstreamUrl: vllm.URL,
|
||||
ApiKey: &staticKey,
|
||||
Enabled: ptr(true),
|
||||
Models: priced(unionA, unionB, unionC),
|
||||
BootstrapCluster: ptr(harness.AgentNetworkCluster),
|
||||
})
|
||||
require.NoError(t, err, "create union provider")
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), p1.Id) })
|
||||
|
||||
polUX, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-mg-union-x",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{gUX.Id},
|
||||
DestinationProviderIds: []string{p1.Id},
|
||||
GuardrailIds: &[]string{mkAllowGuardrail(t, ctx, "e2e-mg-union-x", unionA).Id},
|
||||
})
|
||||
require.NoError(t, err, "create union policy X")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), polUX.Id) })
|
||||
|
||||
polUY, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-mg-union-y",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{gUY.Id},
|
||||
DestinationProviderIds: []string{p1.Id},
|
||||
GuardrailIds: &[]string{mkAllowGuardrail(t, ctx, "e2e-mg-union-y", unionB).Id},
|
||||
})
|
||||
require.NoError(t, err, "create union policy Y")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), polUY.Id) })
|
||||
|
||||
// P2 — mixed scenario: one restricting policy + one un-guardrailed policy.
|
||||
p2, err := srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{
|
||||
Name: "e2e-mg-mix",
|
||||
ProviderId: "openai_api",
|
||||
UpstreamUrl: vllm.URL,
|
||||
ApiKey: &staticKey,
|
||||
Enabled: ptr(true),
|
||||
Models: priced(mixA, mixZ),
|
||||
})
|
||||
require.NoError(t, err, "create mix provider")
|
||||
t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), p2.Id) })
|
||||
|
||||
polMP, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-mg-mix-p",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{gMP.Id},
|
||||
DestinationProviderIds: []string{p2.Id},
|
||||
GuardrailIds: &[]string{mkAllowGuardrail(t, ctx, "e2e-mg-mix-p", mixA).Id},
|
||||
})
|
||||
require.NoError(t, err, "create mix policy P")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), polMP.Id) })
|
||||
|
||||
polMQ, err := srv.CreatePolicy(ctx, api.AgentNetworkPolicyRequest{
|
||||
Name: "e2e-mg-mix-q",
|
||||
Enabled: &enabled,
|
||||
SourceGroups: []string{gMQ.Id},
|
||||
DestinationProviderIds: []string{p2.Id}, // NO guardrail -> unrestricted
|
||||
})
|
||||
require.NoError(t, err, "create mix policy Q")
|
||||
t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), polMQ.Id) })
|
||||
|
||||
settings, err := srv.GetSettings(ctx)
|
||||
require.NoError(t, err, "read settings")
|
||||
require.NotEmpty(t, settings.Endpoint, "endpoint must be assigned")
|
||||
|
||||
proxyToken, err := srv.CreateProxyTokenCLI(ctx, "e2e-mg-proxy")
|
||||
require.NoError(t, err, "mint proxy token")
|
||||
px, err := harness.StartProxy(ctx, srv, proxyToken)
|
||||
require.NoError(t, err, "start proxy")
|
||||
t.Cleanup(func() { _ = px.Terminate(context.Background()) })
|
||||
|
||||
cl, err := harness.StartClient(ctx, srv, sk.Key)
|
||||
require.NoError(t, err, "start client")
|
||||
t.Cleanup(func() { _ = cl.Terminate(context.Background()) })
|
||||
|
||||
require.NoError(t, cl.WaitConnected(ctx, 90*time.Second), "client must connect to management")
|
||||
proxyIP, err := cl.ResolveProxyIP(ctx, settings.Endpoint)
|
||||
require.NoError(t, err, "resolve endpoint to proxy IP")
|
||||
if err := cl.WaitProxyPeer(ctx, 180*time.Second); err != nil {
|
||||
t.Fatalf("client did not see the proxy peer: %v\n=== proxy logs ===\n%s", err, px.Logs(context.Background()))
|
||||
}
|
||||
|
||||
send := func(model string) (int, string) {
|
||||
code, body, cerr := cl.Chat(ctx, settings.Endpoint, proxyIP, harness.WireChat, model, "Reply with exactly: pong", "")
|
||||
require.NoError(t, cerr, "request must reach the proxy")
|
||||
return code, body
|
||||
}
|
||||
sendUntil200 := func(model string) (int, string) {
|
||||
var code int
|
||||
var body string
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
code, body = send(model)
|
||||
if code == 200 {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
return code, body
|
||||
}
|
||||
|
||||
t.Run("union across the user's groups", func(t *testing.T) {
|
||||
code, body := sendUntil200(unionA)
|
||||
assert.Equal(t, 200, code, "model allowed by group X must be served; body: %s\n=== proxy logs ===\n%s", body, px.Logs(context.Background()))
|
||||
|
||||
code, body = sendUntil200(unionB)
|
||||
assert.Equal(t, 200, code, "model allowed by group Y must also be served (union across the user's groups); body: %s", body)
|
||||
|
||||
code, body = send(unionC)
|
||||
assert.Equal(t, 403, code, "a model on neither group's allowlist must be denied; body: %s", body)
|
||||
assert.Contains(t, body, "llm_policy.model_blocked")
|
||||
})
|
||||
|
||||
t.Run("an un-guardrailed group lifts the restriction", func(t *testing.T) {
|
||||
code, body := sendUntil200(mixA)
|
||||
assert.Equal(t, 200, code, "the restricted group's model must be served; body: %s\n=== proxy logs ===\n%s", body, px.Logs(context.Background()))
|
||||
|
||||
code, body = sendUntil200(mixZ)
|
||||
assert.Equal(t, 200, code,
|
||||
"a non-allowlisted model must be served because the user is also in a group whose policy has no guardrail; body: %s\n=== proxy logs ===\n%s", body, px.Logs(context.Background()))
|
||||
})
|
||||
}
|
||||
|
||||
// mkAllowGuardrail creates a guardrail whose model allowlist is enabled and holds
|
||||
// exactly the given model, registering cleanup.
|
||||
func mkAllowGuardrail(t *testing.T, ctx context.Context, name, model string) api.AgentNetworkGuardrail {
|
||||
t.Helper()
|
||||
var gr api.AgentNetworkGuardrailRequest
|
||||
gr.Name = name
|
||||
gr.Checks.ModelAllowlist.Enabled = true
|
||||
gr.Checks.ModelAllowlist.Models = []string{model}
|
||||
g, err := srv.CreateGuardrail(ctx, gr)
|
||||
require.NoError(t, err, "create guardrail %s", name)
|
||||
t.Cleanup(func() { _ = srv.DeleteGuardrail(context.Background(), g.Id) })
|
||||
return g
|
||||
}
|
||||
@@ -38,7 +38,11 @@ type Proxy struct {
|
||||
// network, registered via the given account proxy token and serving the
|
||||
// AgentNetworkCluster over a self-signed wildcard cert. It does not wait for
|
||||
// peer connectivity — callers poll management for the proxy peer.
|
||||
func StartProxy(ctx context.Context, c *Combined, proxyToken string) (*Proxy, error) {
|
||||
// StartProxy launches the reverse-proxy container. Optional envOverrides are
|
||||
// merged into the container environment after the defaults, so callers can set
|
||||
// or override any NB_PROXY_* var (e.g. NB_PROXY_TUNNEL_CACHE_TTL for tests that
|
||||
// need a short authorization-cache window).
|
||||
func StartProxy(ctx context.Context, c *Combined, proxyToken string, envOverrides ...map[string]string) (*Proxy, error) {
|
||||
root, err := repoRoot()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -93,6 +97,12 @@ func StartProxy(ctx context.Context, c *Combined, proxyToken string) (*Proxy, er
|
||||
WaitingFor: wait.ForLog("Initial mapping sync complete").WithStartupTimeout(90 * time.Second),
|
||||
}
|
||||
|
||||
for _, ov := range envOverrides {
|
||||
for k, v := range ov {
|
||||
req.Env[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
|
||||
ContainerRequest: req,
|
||||
Started: true,
|
||||
|
||||
@@ -25,8 +25,8 @@ type Model struct {
|
||||
// These typically need NetBird identity stamped onto upstream
|
||||
// requests so the gateway's analytics and budgets attribute to the
|
||||
// real caller; that's what IdentityInjection is for.
|
||||
// - KindCustom: named and generic OpenAI-compatible self-hosted
|
||||
// endpoints (vLLM, Ollama, custom inference servers).
|
||||
// - KindCustom: the catch-all "OpenAI-compatible self-hosted endpoint"
|
||||
// entry (vLLM, Ollama, custom inference servers).
|
||||
//
|
||||
// Frontend uses Kind to group the provider Select in the modal so an
|
||||
// operator can spot at a glance which catalog entries proxy other
|
||||
@@ -40,18 +40,6 @@ const (
|
||||
KindCustom ProviderKind = "custom"
|
||||
)
|
||||
|
||||
// AuthMode describes whether a catalog provider accepts an upstream
|
||||
// credential. The zero value intentionally resolves to AuthModeRequired so
|
||||
// existing and newly-added catalog entries fail closed unless they explicitly
|
||||
// opt into a less restrictive mode.
|
||||
type AuthMode string
|
||||
|
||||
const (
|
||||
AuthModeRequired AuthMode = "required"
|
||||
AuthModeOptional AuthMode = "optional"
|
||||
AuthModeNone AuthMode = "none"
|
||||
)
|
||||
|
||||
// Provider is the in-memory representation of a catalog provider.
|
||||
type Provider struct {
|
||||
ID string
|
||||
@@ -60,9 +48,6 @@ type Provider struct {
|
||||
DefaultHost string
|
||||
// Kind groups this entry for UI presentation; see ProviderKind.
|
||||
Kind ProviderKind
|
||||
// AuthMode declares whether the provider requires, optionally accepts, or
|
||||
// does not support an upstream API key. Empty defaults to required.
|
||||
AuthMode AuthMode
|
||||
// AuthHeaderName is the HTTP header the provider's API expects
|
||||
// the credential under (e.g. "Authorization" for OpenAI,
|
||||
// "x-api-key" for Anthropic). Combined with AuthHeaderTemplate
|
||||
@@ -101,17 +86,6 @@ type Provider struct {
|
||||
// upstream provider + credentials on Portkey's hosted side).
|
||||
ExtraHeaders []ExtraHeader
|
||||
Models []Model
|
||||
// ModelDiscovery configures provider-specific discovery behavior. A nil
|
||||
// profile means discovery is unsupported. The profile never carries
|
||||
// caller-supplied paths: the proxy owns the fixed endpoint allowlist.
|
||||
ModelDiscovery *ModelDiscovery
|
||||
}
|
||||
|
||||
// ModelDiscovery describes the safe, catalog-owned fallback behavior for a
|
||||
// discoverable provider. Every discovery starts with OpenAI-compatible
|
||||
// /v1/models; OllamaFallback permits /api/tags only when that route is absent.
|
||||
type ModelDiscovery struct {
|
||||
OllamaFallback bool
|
||||
}
|
||||
|
||||
// ExtraHeader names a single optional per-provider routing/config
|
||||
@@ -487,27 +461,6 @@ var providers = []Provider{
|
||||
{ID: "kimi-k3", Label: "Kimi K3", InputPer1k: 0.003, OutputPer1k: 0.015, ContextWindow: 1000000},
|
||||
},
|
||||
},
|
||||
{
|
||||
// Direct Ollama Cloud uses the hosted OpenAI-compatible /v1 API.
|
||||
// Models are discovered dynamically because the Cloud catalog changes
|
||||
// independently of NetBird releases. ParserID intentionally remains
|
||||
// empty to preserve the routing behavior of Ollama, vLLM, and custom
|
||||
// OpenAI-compatible providers.
|
||||
ID: "ollama_cloud",
|
||||
Kind: KindProvider,
|
||||
AuthMode: AuthModeRequired,
|
||||
Name: "Ollama Cloud",
|
||||
Description: "Hosted Ollama models via the OpenAI-compatible API",
|
||||
DefaultHost: "ollama.com",
|
||||
AuthHeaderName: "Authorization",
|
||||
AuthHeaderTemplate: "Bearer ${API_KEY}",
|
||||
DefaultContentType: "application/json",
|
||||
BrandColor: "#000000",
|
||||
Models: []Model{},
|
||||
ModelDiscovery: &ModelDiscovery{
|
||||
OllamaFallback: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "litellm_proxy",
|
||||
Kind: KindGateway,
|
||||
@@ -748,31 +701,11 @@ var providers = []Provider{
|
||||
BrandColor: "#30A2FF",
|
||||
Models: []Model{},
|
||||
},
|
||||
{
|
||||
// Ollama exposes an OpenAI-compatible /v1 API. Like vLLM, it gets a
|
||||
// dedicated catalog id for provider-specific setup guidance while
|
||||
// retaining generic custom-provider routing. Authentication is optional
|
||||
// because local Ollama is authless but protected front ends may use it.
|
||||
ID: "ollama",
|
||||
Kind: KindCustom,
|
||||
AuthMode: AuthModeOptional,
|
||||
Name: "Ollama",
|
||||
Description: "Self-hosted Ollama (OpenAI-compatible)",
|
||||
DefaultHost: "",
|
||||
AuthHeaderName: "Authorization",
|
||||
AuthHeaderTemplate: "Bearer ${API_KEY}",
|
||||
DefaultContentType: "application/json",
|
||||
BrandColor: "#000000",
|
||||
Models: []Model{},
|
||||
ModelDiscovery: &ModelDiscovery{
|
||||
OllamaFallback: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "custom",
|
||||
Kind: KindCustom,
|
||||
Name: "Custom / Self-hosted",
|
||||
Description: "Other OpenAI-compatible endpoint",
|
||||
Description: "OpenAI-compatible endpoint (vLLM, Ollama, …)",
|
||||
DefaultHost: "",
|
||||
AuthHeaderName: "Authorization",
|
||||
AuthHeaderTemplate: "Bearer ${API_KEY}",
|
||||
@@ -805,15 +738,6 @@ func IsKnown(id string) bool {
|
||||
return ok
|
||||
}
|
||||
|
||||
// EffectiveAuthMode returns the provider's configured authentication mode.
|
||||
// Treating the zero value as required keeps older catalog entries fail closed.
|
||||
func (p Provider) EffectiveAuthMode() AuthMode {
|
||||
if p.AuthMode == "" {
|
||||
return AuthModeRequired
|
||||
}
|
||||
return p.AuthMode
|
||||
}
|
||||
|
||||
// IsVertexPathStyle reports whether a provider uses the Google Vertex AI
|
||||
// request shape — the model is carried in the URL path
|
||||
// (/v1/projects/{p}/locations/{r}/publishers/{pub}/models/{model}:{action})
|
||||
@@ -850,17 +774,15 @@ func (p Provider) ToAPIResponse() api.AgentNetworkCatalogProvider {
|
||||
kind = api.AgentNetworkCatalogProviderKindCustom
|
||||
}
|
||||
resp := api.AgentNetworkCatalogProvider{
|
||||
Id: p.ID,
|
||||
Name: p.Name,
|
||||
Description: p.Description,
|
||||
DefaultHost: p.DefaultHost,
|
||||
Kind: kind,
|
||||
AuthMode: api.AgentNetworkCatalogProviderAuthMode(p.EffectiveAuthMode()),
|
||||
AuthHeaderTemplate: p.AuthHeaderTemplate,
|
||||
DefaultContentType: p.DefaultContentType,
|
||||
BrandColor: p.BrandColor,
|
||||
Models: models,
|
||||
SupportsModelDiscovery: p.ModelDiscovery != nil,
|
||||
Id: p.ID,
|
||||
Name: p.Name,
|
||||
Description: p.Description,
|
||||
DefaultHost: p.DefaultHost,
|
||||
Kind: kind,
|
||||
AuthHeaderTemplate: p.AuthHeaderTemplate,
|
||||
DefaultContentType: p.DefaultContentType,
|
||||
BrandColor: p.BrandColor,
|
||||
Models: models,
|
||||
}
|
||||
if len(p.ExtraHeaders) > 0 {
|
||||
extras := make([]api.AgentNetworkCatalogExtraHeader, 0, len(p.ExtraHeaders))
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
func TestOllamaCatalogEntry(t *testing.T) {
|
||||
entry, ok := Lookup("ollama")
|
||||
require.True(t, ok, "Ollama must be available as a dedicated catalog provider")
|
||||
|
||||
assert.Equal(t, KindCustom, entry.Kind)
|
||||
assert.Equal(t, AuthModeOptional, entry.EffectiveAuthMode())
|
||||
assert.Equal(t, "Ollama", entry.Name)
|
||||
assert.Equal(t, "Self-hosted Ollama (OpenAI-compatible)", entry.Description)
|
||||
assert.Empty(t, entry.DefaultHost, "the dashboard owns Ollama's scheme-aware HTTP placeholder")
|
||||
assert.Equal(t, "Authorization", entry.AuthHeaderName)
|
||||
assert.Equal(t, "Bearer ${API_KEY}", entry.AuthHeaderTemplate)
|
||||
assert.Equal(t, "application/json", entry.DefaultContentType)
|
||||
assert.Empty(t, entry.ParserID, "Ollama preserves the untagged vLLM/custom routing behavior")
|
||||
assert.Empty(t, entry.Models, "Ollama models are installed dynamically on the configured endpoint")
|
||||
require.NotNil(t, entry.ModelDiscovery)
|
||||
assert.True(t, entry.ModelDiscovery.OllamaFallback)
|
||||
|
||||
wire := entry.ToAPIResponse()
|
||||
assert.Equal(t, "ollama", wire.Id)
|
||||
assert.Equal(t, api.AgentNetworkCatalogProviderKindCustom, wire.Kind)
|
||||
assert.Equal(t, api.AgentNetworkCatalogProviderAuthModeOptional, wire.AuthMode)
|
||||
assert.True(t, wire.SupportsModelDiscovery)
|
||||
assert.NotNil(t, wire.Models)
|
||||
assert.Empty(t, wire.Models)
|
||||
}
|
||||
|
||||
func TestOllamaCloudCatalogEntry(t *testing.T) {
|
||||
entry, ok := Lookup("ollama_cloud")
|
||||
require.True(t, ok, "Ollama Cloud must be available as a dedicated catalog provider")
|
||||
|
||||
assert.Equal(t, KindProvider, entry.Kind)
|
||||
assert.Equal(t, AuthModeRequired, entry.EffectiveAuthMode())
|
||||
assert.Equal(t, "Ollama Cloud", entry.Name)
|
||||
assert.Equal(t, "Hosted Ollama models via the OpenAI-compatible API", entry.Description)
|
||||
assert.Equal(t, "ollama.com", entry.DefaultHost)
|
||||
assert.Equal(t, "Authorization", entry.AuthHeaderName)
|
||||
assert.Equal(t, "Bearer ${API_KEY}", entry.AuthHeaderTemplate)
|
||||
assert.Equal(t, "application/json", entry.DefaultContentType)
|
||||
assert.Empty(t, entry.ParserID, "Ollama Cloud preserves the untagged Ollama/vLLM/custom routing behavior")
|
||||
assert.Empty(t, entry.Models, "Ollama Cloud models are discovered dynamically")
|
||||
require.NotNil(t, entry.ModelDiscovery)
|
||||
assert.True(t, entry.ModelDiscovery.OllamaFallback)
|
||||
|
||||
wire := entry.ToAPIResponse()
|
||||
assert.Equal(t, "ollama_cloud", wire.Id)
|
||||
assert.Equal(t, api.AgentNetworkCatalogProviderKindProvider, wire.Kind)
|
||||
assert.Equal(t, api.AgentNetworkCatalogProviderAuthModeRequired, wire.AuthMode)
|
||||
assert.Equal(t, "ollama.com", wire.DefaultHost)
|
||||
assert.True(t, wire.SupportsModelDiscovery)
|
||||
assert.NotNil(t, wire.Models)
|
||||
assert.Empty(t, wire.Models)
|
||||
}
|
||||
|
||||
func TestOnlyOllamaProvidersSupportModelDiscovery(t *testing.T) {
|
||||
discoverable := map[string]bool{
|
||||
"ollama": true,
|
||||
"ollama_cloud": true,
|
||||
}
|
||||
for _, entry := range All() {
|
||||
supportsDiscovery := entry.ModelDiscovery != nil
|
||||
assert.Equal(t, discoverable[entry.ID], supportsDiscovery, entry.ID)
|
||||
assert.Equal(t, supportsDiscovery, entry.ToAPIResponse().SupportsModelDiscovery, entry.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCatalogAuthenticationModes(t *testing.T) {
|
||||
openAI, ok := Lookup("openai_api")
|
||||
require.True(t, ok)
|
||||
assert.Empty(t, openAI.AuthMode, "existing entries use the fail-closed default")
|
||||
assert.Equal(t, AuthModeRequired, openAI.EffectiveAuthMode())
|
||||
assert.Equal(t, api.AgentNetworkCatalogProviderAuthModeRequired, openAI.ToAPIResponse().AuthMode)
|
||||
|
||||
for _, entry := range All() {
|
||||
switch entry.EffectiveAuthMode() {
|
||||
case AuthModeRequired, AuthModeOptional:
|
||||
assert.NotEmpty(t, entry.AuthHeaderName, "%s must declare an auth header name", entry.ID)
|
||||
assert.NotEmpty(t, entry.AuthHeaderTemplate, "%s must declare an auth header template", entry.ID)
|
||||
case AuthModeNone:
|
||||
assert.Empty(t, entry.AuthHeaderName, "%s must not declare an auth header name", entry.ID)
|
||||
assert.Empty(t, entry.AuthHeaderTemplate, "%s must not declare an auth header template", entry.ID)
|
||||
default:
|
||||
t.Errorf("%s has invalid auth mode %q", entry.ID, entry.AuthMode)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
|
||||
agentNetworkTypes "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
|
||||
nbcontext "github.com/netbirdio/netbird/management/server/context"
|
||||
"github.com/netbirdio/netbird/shared/auth"
|
||||
"github.com/netbirdio/netbird/shared/management/http/api"
|
||||
)
|
||||
|
||||
type discoveryManagerStub struct {
|
||||
agentnetwork.Manager
|
||||
result *agentNetworkTypes.ModelDiscoveryResult
|
||||
err error
|
||||
accountID string
|
||||
userID string
|
||||
providerID string
|
||||
}
|
||||
|
||||
func (m *discoveryManagerStub) DiscoverProviderModels(_ context.Context, accountID, userID, providerID string) (*agentNetworkTypes.ModelDiscoveryResult, error) {
|
||||
m.accountID = accountID
|
||||
m.userID = userID
|
||||
m.providerID = providerID
|
||||
return m.result, m.err
|
||||
}
|
||||
|
||||
func TestDiscoverProviderModelsHandler(t *testing.T) {
|
||||
manager := &discoveryManagerStub{
|
||||
Manager: agentnetwork.NewManagerMock(),
|
||||
result: &agentNetworkTypes.ModelDiscoveryResult{
|
||||
RequestID: "probe-123",
|
||||
Source: "ollama_api_tags",
|
||||
ProxyCluster: "private.example.com",
|
||||
Models: []agentNetworkTypes.DiscoveredModel{
|
||||
{ID: "llama3.2:latest", Label: "llama3.2:latest"},
|
||||
},
|
||||
},
|
||||
}
|
||||
router := mux.NewRouter()
|
||||
RegisterEndpoints(manager, router)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/agent-network/providers/provider-1/discover-models", nil)
|
||||
req = nbcontext.SetUserAuthInRequest(req, auth.UserAuth{
|
||||
UserId: testUserID,
|
||||
AccountId: testAccountID,
|
||||
})
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code, rec.Body.String())
|
||||
assert.Equal(t, "no-store", rec.Header().Get("Cache-Control"))
|
||||
var response api.AgentNetworkModelDiscoveryResponse
|
||||
require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &response))
|
||||
assert.Equal(t, testAccountID, manager.accountID)
|
||||
assert.Equal(t, testUserID, manager.userID)
|
||||
assert.Equal(t, "provider-1", manager.providerID)
|
||||
assert.Equal(t, "probe-123", response.RequestId)
|
||||
assert.Equal(t, "private.example.com", response.ProxyCluster)
|
||||
assert.Equal(t, api.AgentNetworkModelDiscoveryResponseSourceOllamaApiTags, response.Source)
|
||||
require.Len(t, response.Models, 1)
|
||||
assert.Equal(t, "llama3.2:latest", response.Models[0].Id)
|
||||
}
|
||||
@@ -35,7 +35,6 @@ func RegisterEndpoints(manager agentnetwork.Manager, router *mux.Router) {
|
||||
router.HandleFunc("/agent-network/providers/{providerId}", h.getProvider).Methods("GET", "OPTIONS")
|
||||
router.HandleFunc("/agent-network/providers/{providerId}", h.updateProvider).Methods("PUT", "OPTIONS")
|
||||
router.HandleFunc("/agent-network/providers/{providerId}", h.deleteProvider).Methods("DELETE", "OPTIONS")
|
||||
router.HandleFunc("/agent-network/providers/{providerId}/discover-models", h.discoverProviderModels).Methods("POST", "OPTIONS")
|
||||
h.addPolicyEndpoints(router)
|
||||
h.addGuardrailEndpoints(router)
|
||||
h.addSettingsEndpoints(router)
|
||||
@@ -99,41 +98,6 @@ func (h *handler) getProvider(w http.ResponseWriter, r *http.Request) {
|
||||
util.WriteJSONObject(r.Context(), w, provider.ToAPIResponse())
|
||||
}
|
||||
|
||||
func (h *handler) discoverProviderModels(w http.ResponseWriter, r *http.Request) {
|
||||
userAuth, err := nbcontext.GetUserAuthFromContext(r.Context())
|
||||
if err != nil {
|
||||
util.WriteError(r.Context(), err, w)
|
||||
return
|
||||
}
|
||||
|
||||
providerID := strings.TrimSpace(mux.Vars(r)["providerId"])
|
||||
if providerID == "" {
|
||||
util.WriteError(r.Context(), status.Errorf(status.InvalidArgument, "provider ID is required"), w)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.manager.DiscoverProviderModels(r.Context(), userAuth.AccountId, userAuth.UserId, providerID)
|
||||
if err != nil {
|
||||
util.WriteError(r.Context(), err, w)
|
||||
return
|
||||
}
|
||||
|
||||
models := make([]api.AgentNetworkDiscoveredModel, 0, len(result.Models))
|
||||
for _, model := range result.Models {
|
||||
models = append(models, api.AgentNetworkDiscoveredModel{
|
||||
Id: model.ID,
|
||||
Label: model.Label,
|
||||
})
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
util.WriteJSONObject(r.Context(), w, api.AgentNetworkModelDiscoveryResponse{
|
||||
Models: models,
|
||||
Source: api.AgentNetworkModelDiscoveryResponseSource(result.Source),
|
||||
ProxyCluster: result.ProxyCluster,
|
||||
RequestId: result.RequestID,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *handler) createProvider(w http.ResponseWriter, r *http.Request) {
|
||||
userAuth, err := nbcontext.GetUserAuthFromContext(r.Context())
|
||||
if err != nil {
|
||||
@@ -229,12 +193,11 @@ func (h *handler) deleteProvider(w http.ResponseWriter, r *http.Request) {
|
||||
util.WriteJSONObject(r.Context(), w, util.EmptyObject{})
|
||||
}
|
||||
|
||||
func validate(req *api.AgentNetworkProviderRequest, creating bool) error {
|
||||
func validate(req *api.AgentNetworkProviderRequest, requireAPIKey bool) error {
|
||||
if strings.TrimSpace(req.ProviderId) == "" {
|
||||
return status.Errorf(status.InvalidArgument, "provider_id is required")
|
||||
}
|
||||
entry, ok := catalog.Lookup(req.ProviderId)
|
||||
if !ok {
|
||||
if !catalog.IsKnown(req.ProviderId) {
|
||||
return status.Errorf(status.InvalidArgument, "provider_id %q is not a known catalog provider", req.ProviderId)
|
||||
}
|
||||
if strings.TrimSpace(req.Name) == "" {
|
||||
@@ -247,27 +210,8 @@ func validate(req *api.AgentNetworkProviderRequest, creating bool) error {
|
||||
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
|
||||
return status.Errorf(status.InvalidArgument, "upstream_url must be a full http(s) URL")
|
||||
}
|
||||
return validateProviderAPIKey(entry, req.ApiKey, creating)
|
||||
}
|
||||
|
||||
func validateProviderAPIKey(entry catalog.Provider, apiKey *string, creating bool) error {
|
||||
apiKeyBlank := apiKey == nil || strings.TrimSpace(*apiKey) == ""
|
||||
switch entry.EffectiveAuthMode() {
|
||||
case catalog.AuthModeRequired:
|
||||
// Updates may omit the key to preserve it, but an explicitly empty
|
||||
// value cannot clear a credential required by the selected provider.
|
||||
if (creating || apiKey != nil) && apiKeyBlank {
|
||||
return status.Errorf(status.InvalidArgument, "api_key is required for provider_id %q", entry.ID)
|
||||
}
|
||||
case catalog.AuthModeOptional:
|
||||
// Both omitted and explicitly empty values are valid. The manager
|
||||
// distinguishes preserve from clear during update.
|
||||
case catalog.AuthModeNone:
|
||||
if !apiKeyBlank {
|
||||
return status.Errorf(status.InvalidArgument, "api_key is not supported for provider_id %q", entry.ID)
|
||||
}
|
||||
default:
|
||||
return status.Errorf(status.InvalidArgument, "provider_id %q has an invalid authentication mode", entry.ID)
|
||||
if requireAPIKey && (req.ApiKey == nil || strings.TrimSpace(*req.ApiKey) == "") {
|
||||
return status.Errorf(status.InvalidArgument, "api_key is required")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/catalog"
|
||||
)
|
||||
|
||||
func TestValidateProviderAPIKey(t *testing.T) {
|
||||
key := "secret"
|
||||
empty := ""
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mode catalog.AuthMode
|
||||
apiKey *string
|
||||
creating bool
|
||||
wantErr string
|
||||
}{
|
||||
{name: "required create with key", mode: catalog.AuthModeRequired, apiKey: &key, creating: true},
|
||||
{name: "required create omitted", mode: catalog.AuthModeRequired, creating: true, wantErr: "required"},
|
||||
{name: "required update omitted preserves", mode: catalog.AuthModeRequired},
|
||||
{name: "required update explicit empty", mode: catalog.AuthModeRequired, apiKey: &empty, wantErr: "required"},
|
||||
{name: "optional create omitted", mode: catalog.AuthModeOptional, creating: true},
|
||||
{name: "optional update explicit empty", mode: catalog.AuthModeOptional, apiKey: &empty},
|
||||
{name: "optional with key", mode: catalog.AuthModeOptional, apiKey: &key, creating: true},
|
||||
{name: "none omitted", mode: catalog.AuthModeNone, creating: true},
|
||||
{name: "none explicit empty", mode: catalog.AuthModeNone, apiKey: &empty},
|
||||
{name: "none with key", mode: catalog.AuthModeNone, apiKey: &key, wantErr: "not supported"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
entry := catalog.Provider{ID: "test-provider", AuthMode: tt.mode}
|
||||
err := validateProviderAPIKey(entry, tt.apiKey, tt.creating)
|
||||
if tt.wantErr == "" {
|
||||
require.NoError(t, err)
|
||||
return
|
||||
}
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tt.wantErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -9,12 +9,9 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/catalog"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/labelgen"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy"
|
||||
@@ -51,7 +48,6 @@ func ensureSessionKeys(p *types.Provider) error {
|
||||
type Manager interface {
|
||||
GetAllProviders(ctx context.Context, accountID, userID string) ([]*types.Provider, error)
|
||||
GetProvider(ctx context.Context, accountID, userID, providerID string) (*types.Provider, error)
|
||||
DiscoverProviderModels(ctx context.Context, accountID, userID, providerID string) (*types.ModelDiscoveryResult, error)
|
||||
CreateProvider(ctx context.Context, userID string, provider *types.Provider, bootstrapCluster string) (*types.Provider, error)
|
||||
UpdateProvider(ctx context.Context, userID string, provider *types.Provider) (*types.Provider, error)
|
||||
DeleteProvider(ctx context.Context, accountID, userID, providerID string) error
|
||||
@@ -90,12 +86,17 @@ type Manager interface {
|
||||
|
||||
// PolicySelectionInput is the per-request selection envelope. The
|
||||
// proxy populates it from CapturedData (account, user, groups) plus
|
||||
// the provider llm_router resolved.
|
||||
// the provider llm_router resolved and the model it extracted.
|
||||
type PolicySelectionInput struct {
|
||||
AccountID string
|
||||
UserID string
|
||||
GroupIDs []string
|
||||
ProviderID string
|
||||
// Model is the already-normalised upstream model id the proxy extracted
|
||||
// (parser strips Bedrock region/version, Vertex @version), so a
|
||||
// case-insensitive compare suffices. Empty = undetermined → not permitted
|
||||
// (fail closed).
|
||||
Model string
|
||||
}
|
||||
|
||||
// PolicySelectionResult names the policy that "pays" for this request
|
||||
@@ -132,27 +133,8 @@ type managerImpl struct {
|
||||
// state; concurrent provider creates would otherwise race.
|
||||
labelRngMu sync.Mutex
|
||||
labelRng *rand.Rand
|
||||
|
||||
// discoveryAttempts provides lightweight per-provider admission control for
|
||||
// the explicit endpoint probe. It prevents duplicate clicks from occupying
|
||||
// multiple proxy control-stream requests at once and adds a short cooldown
|
||||
// after each attempt.
|
||||
discoveryMu sync.Mutex
|
||||
discoveryAttempts map[string]modelDiscoveryAttempt
|
||||
}
|
||||
|
||||
type modelDiscoveryAttempt struct {
|
||||
inFlight bool
|
||||
lastStarted time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
modelDiscoveryTimeout = 10 * time.Second
|
||||
modelDiscoveryCooldown = 2 * time.Second
|
||||
maxDiscoveredModels = 500
|
||||
maxDiscoveredModelLen = 512
|
||||
)
|
||||
|
||||
// NewManager constructs the persistent Agent Network manager. The
|
||||
// manager persists provider/policy/guardrail configuration and, on
|
||||
// every mutation, reconciles the in-memory synthesised reverse-proxy
|
||||
@@ -171,7 +153,6 @@ func NewManager(
|
||||
proxyController: proxyController,
|
||||
reconcileCache: make(map[string]map[string]*proto.ProxyMapping),
|
||||
labelRng: rand.New(rand.NewSource(time.Now().UnixNano())),
|
||||
discoveryAttempts: make(map[string]modelDiscoveryAttempt),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -189,202 +170,6 @@ func (m *managerImpl) GetProvider(ctx context.Context, accountID, userID, provid
|
||||
return m.store.GetAgentNetworkProviderByID(ctx, store.LockingStrengthNone, accountID, providerID)
|
||||
}
|
||||
|
||||
// DiscoverProviderModels asks a capable proxy in the account's selected
|
||||
// cluster to query the persisted provider endpoint. The browser supplies only
|
||||
// the provider id: URL, TLS policy, and credential are loaded here so a caller
|
||||
// cannot turn this operation into an arbitrary network probe.
|
||||
func (m *managerImpl) DiscoverProviderModels(ctx context.Context, accountID, userID, providerID string) (*types.ModelDiscoveryResult, error) {
|
||||
if err := m.requirePermission(ctx, accountID, userID, operations.Update); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
providerID = strings.TrimSpace(providerID)
|
||||
if providerID == "" {
|
||||
return nil, status.Errorf(status.InvalidArgument, "provider ID is required")
|
||||
}
|
||||
|
||||
provider, err := m.store.GetAgentNetworkProviderByID(ctx, store.LockingStrengthNone, accountID, providerID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entry, ok := catalog.Lookup(provider.ProviderID)
|
||||
if !ok {
|
||||
return nil, status.Errorf(status.PreconditionFailed, "provider references an unknown catalog provider")
|
||||
}
|
||||
if entry.ModelDiscovery == nil {
|
||||
return nil, status.Errorf(status.PreconditionFailed, "provider type %q does not support model discovery", provider.ProviderID)
|
||||
}
|
||||
if m.proxyController == nil {
|
||||
return nil, status.Errorf(status.PreconditionFailed, "model discovery is unavailable")
|
||||
}
|
||||
|
||||
settings, err := m.store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, accountID)
|
||||
if err != nil {
|
||||
var statusErr *status.Error
|
||||
if errors.As(err, &statusErr) && statusErr.Type() == status.NotFound {
|
||||
return nil, status.Errorf(status.PreconditionFailed, "configure an Agent Network proxy cluster before discovering models")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
cluster := strings.TrimSpace(settings.Cluster)
|
||||
if cluster == "" {
|
||||
return nil, status.Errorf(status.PreconditionFailed, "configure an Agent Network proxy cluster before discovering models")
|
||||
}
|
||||
|
||||
authHeaderName, authHeaderValue, gcpKey, err := providerAuthHeader(provider)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(status.PreconditionFailed, "provider authentication is not configured correctly")
|
||||
}
|
||||
if gcpKey != "" {
|
||||
return nil, status.Errorf(status.PreconditionFailed, "provider authentication is not supported for model discovery")
|
||||
}
|
||||
|
||||
attemptKey := accountID + "\x00" + provider.ID
|
||||
if !m.beginModelDiscovery(attemptKey, time.Now()) {
|
||||
return nil, status.Errorf(status.TooManyRequests, "model discovery is already running or was requested too recently")
|
||||
}
|
||||
defer m.finishModelDiscovery(attemptKey)
|
||||
|
||||
probeCtx, cancel := context.WithTimeout(ctx, modelDiscoveryTimeout)
|
||||
defer cancel()
|
||||
probeResult, err := m.proxyController.DiscoverModels(probeCtx, accountID, cluster, &proto.ModelDiscoveryRequest{
|
||||
UpstreamUrl: strings.TrimSpace(provider.UpstreamURL),
|
||||
AuthHeaderName: authHeaderName,
|
||||
AuthHeaderValue: authHeaderValue,
|
||||
SkipTlsVerify: provider.SkipTLSVerification,
|
||||
OllamaFallback: entry.ModelDiscovery.OllamaFallback,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(probeCtx.Err(), context.DeadlineExceeded) {
|
||||
return nil, status.Errorf(status.PreconditionFailed, "model discovery timed out")
|
||||
}
|
||||
if _, ok := status.FromError(err); ok {
|
||||
return nil, err
|
||||
}
|
||||
return nil, status.Errorf(status.PreconditionFailed, "model discovery could not be completed")
|
||||
}
|
||||
if probeResult == nil {
|
||||
return nil, status.Errorf(status.PreconditionFailed, "proxy returned an empty model discovery response")
|
||||
}
|
||||
if strings.TrimSpace(probeResult.Error) != "" {
|
||||
message := safeModelDiscoveryText(probeResult.Error, 256)
|
||||
if message == "" {
|
||||
message = "proxy reported a discovery failure"
|
||||
}
|
||||
requestID := safeModelDiscoveryText(probeResult.RequestId, 128)
|
||||
if requestID != "" {
|
||||
return nil, status.Errorf(status.PreconditionFailed, "model discovery failed: %s (request_id: %s)", message, requestID)
|
||||
}
|
||||
return nil, status.Errorf(status.PreconditionFailed, "model discovery failed: %s", message)
|
||||
}
|
||||
|
||||
requestID := safeModelDiscoveryText(probeResult.RequestId, 128)
|
||||
if requestID == "" {
|
||||
return nil, status.Errorf(status.PreconditionFailed, "proxy returned an uncorrelated model discovery response")
|
||||
}
|
||||
source := strings.TrimSpace(probeResult.Source)
|
||||
switch source {
|
||||
case "openai_v1_models", "ollama_api_tags":
|
||||
default:
|
||||
return nil, status.Errorf(status.PreconditionFailed, "proxy returned an unsupported model discovery response")
|
||||
}
|
||||
|
||||
return &types.ModelDiscoveryResult{
|
||||
RequestID: requestID,
|
||||
Source: source,
|
||||
ProxyCluster: cluster,
|
||||
Models: normalizeDiscoveredModels(probeResult.Models),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *managerImpl) beginModelDiscovery(key string, now time.Time) bool {
|
||||
m.discoveryMu.Lock()
|
||||
defer m.discoveryMu.Unlock()
|
||||
if m.discoveryAttempts == nil {
|
||||
m.discoveryAttempts = make(map[string]modelDiscoveryAttempt)
|
||||
}
|
||||
attempt := m.discoveryAttempts[key]
|
||||
if attempt.inFlight || (!attempt.lastStarted.IsZero() && now.Sub(attempt.lastStarted) < modelDiscoveryCooldown) {
|
||||
return false
|
||||
}
|
||||
attempt.inFlight = true
|
||||
attempt.lastStarted = now
|
||||
m.discoveryAttempts[key] = attempt
|
||||
return true
|
||||
}
|
||||
|
||||
func (m *managerImpl) finishModelDiscovery(key string) {
|
||||
m.discoveryMu.Lock()
|
||||
attempt, ok := m.discoveryAttempts[key]
|
||||
if !ok {
|
||||
m.discoveryMu.Unlock()
|
||||
return
|
||||
}
|
||||
attempt.inFlight = false
|
||||
m.discoveryAttempts[key] = attempt
|
||||
m.discoveryMu.Unlock()
|
||||
|
||||
remaining := time.Until(attempt.lastStarted.Add(modelDiscoveryCooldown))
|
||||
if remaining <= 0 {
|
||||
m.expireModelDiscoveryAttempt(key, attempt.lastStarted)
|
||||
return
|
||||
}
|
||||
time.AfterFunc(remaining, func() {
|
||||
m.expireModelDiscoveryAttempt(key, attempt.lastStarted)
|
||||
})
|
||||
}
|
||||
|
||||
func (m *managerImpl) expireModelDiscoveryAttempt(key string, lastStarted time.Time) {
|
||||
m.discoveryMu.Lock()
|
||||
defer m.discoveryMu.Unlock()
|
||||
attempt, ok := m.discoveryAttempts[key]
|
||||
if ok && !attempt.inFlight && attempt.lastStarted.Equal(lastStarted) {
|
||||
delete(m.discoveryAttempts, key)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeDiscoveredModels(models []*proto.ModelDiscoveryModel) []types.DiscoveredModel {
|
||||
out := make([]types.DiscoveredModel, 0, min(len(models), maxDiscoveredModels))
|
||||
seen := make(map[string]struct{}, min(len(models), maxDiscoveredModels))
|
||||
for _, model := range models {
|
||||
if model == nil {
|
||||
continue
|
||||
}
|
||||
id := safeModelDiscoveryText(model.Id, maxDiscoveredModelLen)
|
||||
if id == "" || len(id) > maxDiscoveredModelLen {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
label := safeModelDiscoveryText(model.Label, maxDiscoveredModelLen)
|
||||
if label == "" || len(label) > maxDiscoveredModelLen {
|
||||
label = id
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
out = append(out, types.DiscoveredModel{ID: id, Label: label})
|
||||
if len(out) == maxDiscoveredModels {
|
||||
break
|
||||
}
|
||||
}
|
||||
slices.SortFunc(out, func(a, b types.DiscoveredModel) int {
|
||||
return strings.Compare(a.ID, b.ID)
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func safeModelDiscoveryText(value string, maxLen int) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" || len(value) > maxLen || !utf8.ValidString(value) {
|
||||
return ""
|
||||
}
|
||||
for _, r := range value {
|
||||
if unicode.IsControl(r) {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// CreateProvider persists a new provider for the account. bootstrapCluster
|
||||
// is used only when the per-account agent-network Settings row hasn't
|
||||
// been created yet; otherwise it is ignored (the cluster is pinned on
|
||||
@@ -394,8 +179,11 @@ func (m *managerImpl) CreateProvider(ctx context.Context, userID string, provide
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := prepareProviderAPIKey(provider, nil); err != nil {
|
||||
return nil, err
|
||||
// An empty api_key would silently produce a synthesised service
|
||||
// that 401s on every upstream request. Surface the misconfiguration
|
||||
// at create time instead.
|
||||
if strings.TrimSpace(provider.APIKey) == "" {
|
||||
return nil, status.Errorf(status.InvalidArgument, "api_key is required when creating an agent network provider")
|
||||
}
|
||||
|
||||
if provider.ID == "" {
|
||||
@@ -439,8 +227,13 @@ func (m *managerImpl) UpdateProvider(ctx context.Context, userID string, provide
|
||||
return nil, fmt.Errorf("failed to get agent network provider: %w", err)
|
||||
}
|
||||
|
||||
if err := prepareProviderAPIKey(provider, existing); err != nil {
|
||||
return nil, err
|
||||
// Preserve the API key if the caller didn't rotate it. A
|
||||
// whitespace-only value is treated as "not rotated" rather than a
|
||||
// real key, but it must not silently overwrite a valid stored key.
|
||||
if provider.APIKey == "" {
|
||||
provider.APIKey = existing.APIKey
|
||||
} else if strings.TrimSpace(provider.APIKey) == "" {
|
||||
return nil, status.Errorf(status.InvalidArgument, "api_key must be non-blank when rotating an agent network provider")
|
||||
}
|
||||
// Always preserve the session keypair across updates so existing
|
||||
// session cookies stay valid. The keys are server-managed and
|
||||
@@ -463,52 +256,6 @@ func (m *managerImpl) UpdateProvider(ctx context.Context, userID string, provide
|
||||
return provider, nil
|
||||
}
|
||||
|
||||
// prepareProviderAPIKey applies catalog-owned authentication semantics before
|
||||
// a provider is persisted. existing is nil on create. On update, omission
|
||||
// preserves a key only when the provider type is unchanged; switching types
|
||||
// never carries an old provider's secret into the new upstream.
|
||||
func prepareProviderAPIKey(provider, existing *types.Provider) error {
|
||||
entry, ok := catalog.Lookup(provider.ProviderID)
|
||||
if !ok {
|
||||
return status.Errorf(status.InvalidArgument, "provider_id %q is not a known catalog provider", provider.ProviderID)
|
||||
}
|
||||
return prepareProviderAPIKeyForEntry(provider, existing, entry)
|
||||
}
|
||||
|
||||
func prepareProviderAPIKeyForEntry(provider, existing *types.Provider, entry catalog.Provider) error {
|
||||
keyProvided := provider.APIKeyProvided || provider.APIKey != ""
|
||||
providerChanged := existing != nil && provider.ProviderID != existing.ProviderID
|
||||
if existing != nil && !keyProvided && !providerChanged && entry.EffectiveAuthMode() != catalog.AuthModeNone {
|
||||
provider.APIKey = existing.APIKey
|
||||
}
|
||||
// Normalize after a preserved value is restored so a legacy
|
||||
// whitespace-only credential cannot pass manager validation and then fail
|
||||
// later during synthesis.
|
||||
if strings.TrimSpace(provider.APIKey) == "" {
|
||||
provider.APIKey = ""
|
||||
}
|
||||
|
||||
switch entry.EffectiveAuthMode() {
|
||||
case catalog.AuthModeRequired:
|
||||
if provider.APIKey == "" {
|
||||
return status.Errorf(status.InvalidArgument, "api_key is required for provider_id %q", provider.ProviderID)
|
||||
}
|
||||
case catalog.AuthModeOptional:
|
||||
// Empty is a valid credential state.
|
||||
case catalog.AuthModeNone:
|
||||
if keyProvided && provider.APIKey != "" {
|
||||
return status.Errorf(status.InvalidArgument, "api_key is not supported for provider_id %q", provider.ProviderID)
|
||||
}
|
||||
// Clear any stale credential already stored for a provider whose
|
||||
// catalog semantics changed to no authentication.
|
||||
provider.APIKey = ""
|
||||
default:
|
||||
return status.Errorf(status.InvalidArgument, "provider_id %q has an invalid authentication mode", provider.ProviderID)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *managerImpl) DeleteProvider(ctx context.Context, accountID, userID, providerID string) error {
|
||||
if err := m.requirePermission(ctx, accountID, userID, operations.Delete); err != nil {
|
||||
return err
|
||||
@@ -1066,10 +813,6 @@ func (*mockManager) GetProvider(_ context.Context, _, _, _ string) (*types.Provi
|
||||
return &types.Provider{}, nil
|
||||
}
|
||||
|
||||
func (*mockManager) DiscoverProviderModels(_ context.Context, _, _, _ string) (*types.ModelDiscoveryResult, error) {
|
||||
return nil, status.Errorf(status.PreconditionFailed, "model discovery is unavailable")
|
||||
}
|
||||
|
||||
func (*mockManager) CreateProvider(_ context.Context, _ string, p *types.Provider, _ string) (*types.Provider, error) {
|
||||
return p, nil
|
||||
}
|
||||
|
||||
@@ -1,248 +0,0 @@
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/golang/mock/gomock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy"
|
||||
"github.com/netbirdio/netbird/management/server/permissions"
|
||||
"github.com/netbirdio/netbird/management/server/permissions/modules"
|
||||
"github.com/netbirdio/netbird/management/server/permissions/operations"
|
||||
"github.com/netbirdio/netbird/management/server/store"
|
||||
"github.com/netbirdio/netbird/shared/management/proto"
|
||||
"github.com/netbirdio/netbird/shared/management/status"
|
||||
)
|
||||
|
||||
func newModelDiscoveryManager(t *testing.T) (*managerImpl, *store.MockStore, *permissions.MockManager, *proxy.MockController) {
|
||||
t.Helper()
|
||||
ctrl := gomock.NewController(t)
|
||||
mockStore := store.NewMockStore(ctrl)
|
||||
mockPermissions := permissions.NewMockManager(ctrl)
|
||||
mockProxy := proxy.NewMockController(ctrl)
|
||||
return &managerImpl{
|
||||
store: mockStore,
|
||||
permissionsManager: mockPermissions,
|
||||
proxyController: mockProxy,
|
||||
discoveryAttempts: make(map[string]modelDiscoveryAttempt),
|
||||
}, mockStore, mockPermissions, mockProxy
|
||||
}
|
||||
|
||||
func allowModelDiscovery(mockPermissions *permissions.MockManager) {
|
||||
mockPermissions.EXPECT().
|
||||
ValidateUserPermissions(gomock.Any(), "account-1", "user-1", modules.AgentNetwork, operations.Update).
|
||||
Return(true, context.Background(), nil)
|
||||
}
|
||||
|
||||
func TestDiscoverProviderModelsUsesPersistedProviderAndCluster(t *testing.T) {
|
||||
manager, mockStore, mockPermissions, mockProxy := newModelDiscoveryManager(t)
|
||||
allowModelDiscovery(mockPermissions)
|
||||
|
||||
provider := &types.Provider{
|
||||
ID: "provider-1",
|
||||
AccountID: "account-1",
|
||||
ProviderID: "ollama",
|
||||
UpstreamURL: "http://ollama.internal:11434/base",
|
||||
APIKey: "stored-key",
|
||||
SkipTLSVerification: true,
|
||||
}
|
||||
mockStore.EXPECT().
|
||||
GetAgentNetworkProviderByID(gomock.Any(), store.LockingStrengthNone, "account-1", "provider-1").
|
||||
Return(provider, nil)
|
||||
mockStore.EXPECT().
|
||||
GetAgentNetworkSettings(gomock.Any(), store.LockingStrengthNone, "account-1").
|
||||
Return(&types.Settings{AccountID: "account-1", Cluster: "private.example.com"}, nil)
|
||||
mockProxy.EXPECT().
|
||||
DiscoverModels(gomock.Any(), "account-1", "private.example.com", gomock.Any()).
|
||||
DoAndReturn(func(_ context.Context, _, _ string, request *proto.ModelDiscoveryRequest) (*proto.ModelDiscoveryResult, error) {
|
||||
assert.Empty(t, request.RequestId, "the control-plane server owns request IDs")
|
||||
assert.Equal(t, provider.UpstreamURL, request.UpstreamUrl)
|
||||
assert.Equal(t, "Authorization", request.AuthHeaderName)
|
||||
assert.Equal(t, "Bearer stored-key", request.AuthHeaderValue)
|
||||
assert.True(t, request.SkipTlsVerify)
|
||||
assert.True(t, request.OllamaFallback)
|
||||
return &proto.ModelDiscoveryResult{
|
||||
RequestId: "probe-123",
|
||||
Source: "openai_v1_models",
|
||||
Models: []*proto.ModelDiscoveryModel{
|
||||
{Id: "zeta", Label: ""},
|
||||
{Id: "alpha", Label: "Alpha"},
|
||||
{Id: "zeta", Label: "duplicate"},
|
||||
{Id: "bad\x00model", Label: "bad"},
|
||||
{Id: strings.Repeat("x", maxDiscoveredModelLen+1), Label: "too long"},
|
||||
},
|
||||
}, nil
|
||||
})
|
||||
|
||||
result, err := manager.DiscoverProviderModels(context.Background(), "account-1", "user-1", "provider-1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "probe-123", result.RequestID)
|
||||
assert.Equal(t, "openai_v1_models", result.Source)
|
||||
assert.Equal(t, "private.example.com", result.ProxyCluster)
|
||||
assert.Equal(t, []types.DiscoveredModel{
|
||||
{ID: "alpha", Label: "Alpha"},
|
||||
{ID: "zeta", Label: "zeta"},
|
||||
}, result.Models)
|
||||
}
|
||||
|
||||
func TestDiscoverProviderModelsSupportsOllamaCloud(t *testing.T) {
|
||||
manager, mockStore, mockPermissions, mockProxy := newModelDiscoveryManager(t)
|
||||
allowModelDiscovery(mockPermissions)
|
||||
|
||||
provider := &types.Provider{
|
||||
ID: "provider-cloud",
|
||||
AccountID: "account-1",
|
||||
ProviderID: "ollama_cloud",
|
||||
UpstreamURL: "https://ollama.com",
|
||||
APIKey: "ollama-cloud-key",
|
||||
}
|
||||
mockStore.EXPECT().
|
||||
GetAgentNetworkProviderByID(gomock.Any(), store.LockingStrengthNone, "account-1", provider.ID).
|
||||
Return(provider, nil)
|
||||
mockStore.EXPECT().
|
||||
GetAgentNetworkSettings(gomock.Any(), store.LockingStrengthNone, "account-1").
|
||||
Return(&types.Settings{AccountID: "account-1", Cluster: "cloud-proxies.example.com"}, nil)
|
||||
mockProxy.EXPECT().
|
||||
DiscoverModels(gomock.Any(), "account-1", "cloud-proxies.example.com", gomock.Any()).
|
||||
DoAndReturn(func(_ context.Context, _, _ string, request *proto.ModelDiscoveryRequest) (*proto.ModelDiscoveryResult, error) {
|
||||
assert.Equal(t, "https://ollama.com", request.UpstreamUrl)
|
||||
assert.Equal(t, "Authorization", request.AuthHeaderName)
|
||||
assert.Equal(t, "Bearer ollama-cloud-key", request.AuthHeaderValue)
|
||||
assert.False(t, request.SkipTlsVerify)
|
||||
assert.True(t, request.OllamaFallback)
|
||||
return &proto.ModelDiscoveryResult{
|
||||
RequestId: "probe-cloud",
|
||||
Source: "openai_v1_models",
|
||||
Models: []*proto.ModelDiscoveryModel{
|
||||
{Id: "gpt-oss:120b", Label: "gpt-oss:120b"},
|
||||
},
|
||||
}, nil
|
||||
})
|
||||
|
||||
result, err := manager.DiscoverProviderModels(context.Background(), "account-1", "user-1", provider.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "probe-cloud", result.RequestID)
|
||||
assert.Equal(t, "cloud-proxies.example.com", result.ProxyCluster)
|
||||
assert.Equal(t, []types.DiscoveredModel{
|
||||
{ID: "gpt-oss:120b", Label: "gpt-oss:120b"},
|
||||
}, result.Models)
|
||||
}
|
||||
|
||||
func TestDiscoverProviderModelsRejectsOllamaCloudWithoutKey(t *testing.T) {
|
||||
manager, mockStore, mockPermissions, _ := newModelDiscoveryManager(t)
|
||||
allowModelDiscovery(mockPermissions)
|
||||
|
||||
provider := &types.Provider{
|
||||
ID: "provider-cloud",
|
||||
AccountID: "account-1",
|
||||
ProviderID: "ollama_cloud",
|
||||
UpstreamURL: "https://ollama.com",
|
||||
}
|
||||
mockStore.EXPECT().
|
||||
GetAgentNetworkProviderByID(gomock.Any(), store.LockingStrengthNone, "account-1", provider.ID).
|
||||
Return(provider, nil)
|
||||
mockStore.EXPECT().
|
||||
GetAgentNetworkSettings(gomock.Any(), store.LockingStrengthNone, "account-1").
|
||||
Return(&types.Settings{AccountID: "account-1", Cluster: "cloud-proxies.example.com"}, nil)
|
||||
|
||||
_, err := manager.DiscoverProviderModels(context.Background(), "account-1", "user-1", provider.ID)
|
||||
require.Error(t, err)
|
||||
statusErr, ok := status.FromError(err)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, status.PreconditionFailed, statusErr.Type())
|
||||
assert.Contains(t, err.Error(), "authentication is not configured correctly")
|
||||
}
|
||||
|
||||
func TestDiscoverProviderModelsRejectsUnsupportedProvider(t *testing.T) {
|
||||
manager, mockStore, mockPermissions, _ := newModelDiscoveryManager(t)
|
||||
allowModelDiscovery(mockPermissions)
|
||||
mockStore.EXPECT().
|
||||
GetAgentNetworkProviderByID(gomock.Any(), store.LockingStrengthNone, "account-1", "provider-1").
|
||||
Return(&types.Provider{
|
||||
ID: "provider-1",
|
||||
AccountID: "account-1",
|
||||
ProviderID: "openai_api",
|
||||
}, nil)
|
||||
|
||||
_, err := manager.DiscoverProviderModels(context.Background(), "account-1", "user-1", "provider-1")
|
||||
require.Error(t, err)
|
||||
statusErr, ok := status.FromError(err)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, status.PreconditionFailed, statusErr.Type())
|
||||
assert.Contains(t, err.Error(), "does not support")
|
||||
}
|
||||
|
||||
func TestDiscoverProviderModelsPreservesCorrelatedProxyError(t *testing.T) {
|
||||
manager, mockStore, mockPermissions, mockProxy := newModelDiscoveryManager(t)
|
||||
allowModelDiscovery(mockPermissions)
|
||||
mockStore.EXPECT().
|
||||
GetAgentNetworkProviderByID(gomock.Any(), store.LockingStrengthNone, "account-1", "provider-1").
|
||||
Return(&types.Provider{
|
||||
ID: "provider-1",
|
||||
AccountID: "account-1",
|
||||
ProviderID: "ollama",
|
||||
UpstreamURL: "http://ollama.internal:11434",
|
||||
}, nil)
|
||||
mockStore.EXPECT().
|
||||
GetAgentNetworkSettings(gomock.Any(), store.LockingStrengthNone, "account-1").
|
||||
Return(&types.Settings{AccountID: "account-1", Cluster: "private.example.com"}, nil)
|
||||
mockProxy.EXPECT().
|
||||
DiscoverModels(gomock.Any(), "account-1", "private.example.com", gomock.Any()).
|
||||
Return(&proto.ModelDiscoveryResult{
|
||||
RequestId: "probe-failed",
|
||||
Error: "upstream returned HTTP 401",
|
||||
}, nil)
|
||||
|
||||
_, err := manager.DiscoverProviderModels(context.Background(), "account-1", "user-1", "provider-1")
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "upstream returned HTTP 401")
|
||||
assert.Contains(t, err.Error(), "probe-failed")
|
||||
}
|
||||
|
||||
func TestModelDiscoveryAdmissionControl(t *testing.T) {
|
||||
manager := &managerImpl{}
|
||||
start := time.Now()
|
||||
require.True(t, manager.beginModelDiscovery("account/provider", start))
|
||||
assert.False(t, manager.beginModelDiscovery("account/provider", start.Add(time.Second)))
|
||||
|
||||
manager.finishModelDiscovery("account/provider")
|
||||
assert.False(t, manager.beginModelDiscovery("account/provider", start.Add(time.Second)))
|
||||
assert.True(t, manager.beginModelDiscovery("account/provider", start.Add(modelDiscoveryCooldown)))
|
||||
}
|
||||
|
||||
func TestModelDiscoveryAdmissionControlExpiresFinishedAttempts(t *testing.T) {
|
||||
manager := &managerImpl{}
|
||||
start := time.Now()
|
||||
require.True(t, manager.beginModelDiscovery("account/provider", start))
|
||||
manager.finishModelDiscovery("account/provider")
|
||||
|
||||
manager.expireModelDiscoveryAttempt("account/provider", start)
|
||||
|
||||
manager.discoveryMu.Lock()
|
||||
_, retained := manager.discoveryAttempts["account/provider"]
|
||||
manager.discoveryMu.Unlock()
|
||||
assert.False(t, retained)
|
||||
}
|
||||
|
||||
func TestModelDiscoveryAdmissionControlKeepsReplacementAttempt(t *testing.T) {
|
||||
manager := &managerImpl{}
|
||||
start := time.Now()
|
||||
require.True(t, manager.beginModelDiscovery("account/provider", start))
|
||||
manager.finishModelDiscovery("account/provider")
|
||||
require.True(t, manager.beginModelDiscovery("account/provider", start.Add(modelDiscoveryCooldown)))
|
||||
|
||||
manager.expireModelDiscoveryAttempt("account/provider", start)
|
||||
|
||||
manager.discoveryMu.Lock()
|
||||
attempt, retained := manager.discoveryAttempts["account/provider"]
|
||||
manager.discoveryMu.Unlock()
|
||||
require.True(t, retained)
|
||||
assert.True(t, attempt.inFlight)
|
||||
assert.Equal(t, start.Add(modelDiscoveryCooldown), attempt.lastStarted)
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
|
||||
@@ -35,6 +36,10 @@ const (
|
||||
denyCodeAccountTokenCapExceeded = "llm_account.token_cap_exceeded"
|
||||
//nolint:gosec // account deny code label, not a credential
|
||||
denyCodeAccountBudgetCapExceeded = "llm_account.budget_cap_exceeded"
|
||||
// denyCodeModelBlocked is returned when policies govern the request's
|
||||
// (provider, caller-groups) but none permits the model. Matches the proxy
|
||||
// guardrail's code so both layers surface the same label.
|
||||
denyCodeModelBlocked = "llm_policy.model_blocked"
|
||||
)
|
||||
|
||||
// consumptionCache holds the consumption counters prefetched for one
|
||||
@@ -159,6 +164,25 @@ func (m *managerImpl) SelectPolicyForRequest(ctx context.Context, in PolicySelec
|
||||
}
|
||||
candidates := filterApplicablePolicies(policies, in)
|
||||
|
||||
// Model-allowlist gate scoped to the matched policies: keep candidates whose
|
||||
// guardrails permit the model (none enabled = unrestricted), deny when
|
||||
// policies apply but none permits it. Skip the load when none has a guardrail.
|
||||
if len(candidates) > 0 && anyPolicyHasGuardrails(candidates) {
|
||||
guardrailsByID, gErr := m.loadGuardrailsByID(ctx, in.AccountID)
|
||||
if gErr != nil {
|
||||
return nil, gErr
|
||||
}
|
||||
permitted := filterModelPermittedPolicies(candidates, guardrailsByID, in.Model)
|
||||
if len(permitted) == 0 {
|
||||
return &PolicySelectionResult{
|
||||
Allow: false,
|
||||
DenyCode: denyCodeModelBlocked,
|
||||
DenyReason: modelBlockedReason(in.Model),
|
||||
}, nil
|
||||
}
|
||||
candidates = permitted
|
||||
}
|
||||
|
||||
// Prefetch every consumption counter the ceiling + candidate policies will
|
||||
// read, in a single store round-trip, then score against the cache.
|
||||
cache, err := m.prefetchConsumption(ctx, in, rules, candidates, now)
|
||||
@@ -250,6 +274,90 @@ func filterApplicablePolicies(policies []*types.Policy, in PolicySelectionInput)
|
||||
return out
|
||||
}
|
||||
|
||||
// anyPolicyHasGuardrails reports whether any policy references at least one
|
||||
// guardrail, so the selector can skip loading guardrails when none do.
|
||||
func anyPolicyHasGuardrails(policies []*types.Policy) bool {
|
||||
for _, p := range policies {
|
||||
if p != nil && len(p.GuardrailIDs) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// loadGuardrailsByID loads the account's guardrails indexed by ID. Used by the
|
||||
// model-allowlist gate to resolve each candidate policy's attached guardrails.
|
||||
func (m *managerImpl) loadGuardrailsByID(ctx context.Context, accountID string) (map[string]*types.Guardrail, error) {
|
||||
guardrails, err := m.store.GetAccountAgentNetworkGuardrails(ctx, store.LockingStrengthNone, accountID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list account guardrails: %w", err)
|
||||
}
|
||||
byID := make(map[string]*types.Guardrail, len(guardrails))
|
||||
for _, g := range guardrails {
|
||||
if g != nil {
|
||||
byID[g.ID] = g
|
||||
}
|
||||
}
|
||||
return byID, nil
|
||||
}
|
||||
|
||||
// filterModelPermittedPolicies returns the subset of policies whose guardrails
|
||||
// permit the model. Order is preserved so downstream scoring is unaffected.
|
||||
func filterModelPermittedPolicies(policies []*types.Policy, byID map[string]*types.Guardrail, model string) []*types.Policy {
|
||||
out := make([]*types.Policy, 0, len(policies))
|
||||
for _, p := range policies {
|
||||
if policyPermitsModel(p, byID, model) {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// policyPermitsModel reports whether a policy permits the model. No
|
||||
// allowlist-enabled guardrail = unrestricted (permits any, incl. empty);
|
||||
// otherwise the model must be in the union of its allowlists, so an
|
||||
// empty/undetermined model fails closed.
|
||||
func policyPermitsModel(p *types.Policy, byID map[string]*types.Guardrail, model string) bool {
|
||||
if p == nil {
|
||||
return false
|
||||
}
|
||||
wanted := normaliseModelID(model)
|
||||
restricted := false
|
||||
for _, gID := range p.GuardrailIDs {
|
||||
g, ok := byID[gID]
|
||||
if !ok || g == nil || !g.Checks.ModelAllowlist.Enabled {
|
||||
continue
|
||||
}
|
||||
restricted = true
|
||||
if wanted == "" {
|
||||
continue
|
||||
}
|
||||
for _, allowed := range g.Checks.ModelAllowlist.Models {
|
||||
if normaliseModelID(allowed) == wanted {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return !restricted
|
||||
}
|
||||
|
||||
// normaliseModelID lowercases and trims a model identifier so the allowlist
|
||||
// compare is case-insensitive and trim-tolerant. Mirrors the proxy guardrail's
|
||||
// normaliseModel so both layers agree on what "same model" means.
|
||||
func normaliseModelID(model string) string {
|
||||
return strings.ToLower(strings.TrimSpace(model))
|
||||
}
|
||||
|
||||
// modelBlockedReason builds the human-readable deny reason for a model-allowlist
|
||||
// rejection. The model is quoted when known; an undetermined model is reported
|
||||
// as such so the access log distinguishes "wrong model" from "no model".
|
||||
func modelBlockedReason(model string) string {
|
||||
if normaliseModelID(model) == "" {
|
||||
return "request model could not be determined for the policy allowlist"
|
||||
}
|
||||
return fmt.Sprintf("model %q is not permitted by any applicable policy allowlist", model)
|
||||
}
|
||||
|
||||
// candidate is the per-policy intermediate the selector ranks. A
|
||||
// policy that's been exhausted on any enabled cap never makes it
|
||||
// into this slice; the selector's deny envelope carries the latest
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/golang/mock/gomock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
|
||||
"github.com/netbirdio/netbird/management/server/store"
|
||||
)
|
||||
|
||||
// guardedPolicy builds an enabled, uncapped policy that authorises sourceGroups
|
||||
// to reach providerID under the given guardrails. Uncapped keeps the selector's
|
||||
// headroom scoring trivial so these tests isolate the model-allowlist gate.
|
||||
func guardedPolicy(id, account string, sourceGroups []string, providerID string, guardrailIDs ...string) *types.Policy {
|
||||
return &types.Policy{
|
||||
ID: id,
|
||||
AccountID: account,
|
||||
Enabled: true,
|
||||
SourceGroups: sourceGroups,
|
||||
DestinationProviderIDs: []string{providerID},
|
||||
GuardrailIDs: guardrailIDs,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
}
|
||||
}
|
||||
|
||||
// allowlistGuardrail builds a guardrail whose model allowlist is enabled and
|
||||
// carries the given models.
|
||||
func allowlistGuardrail(id, account string, models ...string) *types.Guardrail {
|
||||
return &types.Guardrail{
|
||||
ID: id,
|
||||
AccountID: account,
|
||||
Checks: types.GuardrailChecks{
|
||||
ModelAllowlist: types.GuardrailModelAllowlist{Enabled: true, Models: models},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func expectPolicies(mockStore *store.MockStore, account string, policies ...*types.Policy) {
|
||||
mockStore.EXPECT().
|
||||
GetAccountAgentNetworkPolicies(gomock.Any(), gomock.Any(), account).
|
||||
Return(policies, nil)
|
||||
}
|
||||
|
||||
func expectGuardrails(mockStore *store.MockStore, account string, guardrails ...*types.Guardrail) {
|
||||
mockStore.EXPECT().
|
||||
GetAccountAgentNetworkGuardrails(gomock.Any(), gomock.Any(), account).
|
||||
Return(guardrails, nil)
|
||||
}
|
||||
|
||||
// TestSelectPolicy_ModelBlockedByAllowlist proves the authoritative allowlist
|
||||
// decision: a policy authorises the (provider, group) but restricts the model,
|
||||
// and the requested model isn't on the list, so the request is denied.
|
||||
func TestSelectPolicy_ModelBlockedByAllowlist(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
mgr, mockStore := newSelectorMgr(t, ctrl)
|
||||
|
||||
policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1")
|
||||
expectPolicies(mockStore, "acc-1", policy)
|
||||
expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "gpt-4o"))
|
||||
|
||||
res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{
|
||||
AccountID: "acc-1",
|
||||
UserID: "user-1",
|
||||
GroupIDs: []string{"grp-eng"},
|
||||
ProviderID: "prov-1",
|
||||
Model: "claude-opus-4",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.False(t, res.Allow, "a model outside the only applicable policy's allowlist must be denied")
|
||||
assert.Equal(t, denyCodeModelBlocked, res.DenyCode, "deny code must be model_blocked")
|
||||
assert.NotEmpty(t, res.DenyReason, "deny reason must be populated")
|
||||
}
|
||||
|
||||
// TestSelectPolicy_ModelAllowedByAllowlist is the allow counterpart: the model
|
||||
// is on the applicable policy's allowlist, so selection proceeds normally.
|
||||
func TestSelectPolicy_ModelAllowedByAllowlist(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
mgr, mockStore := newSelectorMgr(t, ctrl)
|
||||
|
||||
policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1")
|
||||
expectPolicies(mockStore, "acc-1", policy)
|
||||
expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "gpt-4o", "claude-opus-4"))
|
||||
expectConsumptionBatch(mockStore, nil)
|
||||
|
||||
res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{
|
||||
AccountID: "acc-1",
|
||||
UserID: "user-1",
|
||||
GroupIDs: []string{"grp-eng"},
|
||||
ProviderID: "prov-1",
|
||||
Model: "claude-opus-4",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, res.Allow, "a model on the applicable policy's allowlist must be allowed")
|
||||
assert.Equal(t, "pol-A", res.SelectedPolicyID)
|
||||
}
|
||||
|
||||
// TestSelectPolicy_CaseInsensitiveModelMatch proves the compare tolerates case
|
||||
// and surrounding whitespace, matching the proxy guardrail's normalisation.
|
||||
func TestSelectPolicy_CaseInsensitiveModelMatch(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
mgr, mockStore := newSelectorMgr(t, ctrl)
|
||||
|
||||
policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1")
|
||||
expectPolicies(mockStore, "acc-1", policy)
|
||||
expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", " GPT-4o "))
|
||||
expectConsumptionBatch(mockStore, nil)
|
||||
|
||||
res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{
|
||||
AccountID: "acc-1",
|
||||
GroupIDs: []string{"grp-eng"},
|
||||
ProviderID: "prov-1",
|
||||
Model: "gpt-4o",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, res.Allow, "case/whitespace variants must match the allowlist entry")
|
||||
}
|
||||
|
||||
// TestSelectPolicy_UnguardedPolicyIsUnrestricted is the false-deny fix: when two
|
||||
// policies authorise the same (provider, group) and one has no guardrail, that
|
||||
// policy makes the request unrestricted — not caught by the other's allowlist.
|
||||
func TestSelectPolicy_UnguardedPolicyIsUnrestricted(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
mgr, mockStore := newSelectorMgr(t, ctrl)
|
||||
|
||||
restricted := guardedPolicy("pol-restricted", "acc-1", []string{"grp-eng"}, "prov-1", "g-1")
|
||||
open := guardedPolicy("pol-open", "acc-1", []string{"grp-eng"}, "prov-1") // no guardrail
|
||||
expectPolicies(mockStore, "acc-1", restricted, open)
|
||||
expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "gpt-4o"))
|
||||
expectConsumptionBatch(mockStore, nil)
|
||||
|
||||
res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{
|
||||
AccountID: "acc-1",
|
||||
GroupIDs: []string{"grp-eng"},
|
||||
ProviderID: "prov-1",
|
||||
Model: "claude-opus-4",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, res.Allow, "an un-guardrailed policy for the same (provider, group) must leave the request unrestricted")
|
||||
assert.Equal(t, "pol-open", res.SelectedPolicyID, "the unrestricted policy must be the one that pays")
|
||||
}
|
||||
|
||||
// TestSelectPolicy_AllowlistDoesNotLeakAcrossGroups is the false-allow fix: a
|
||||
// model allowlisted only for grp-b must not be usable by a grp-a caller. The
|
||||
// selector considers only policies applicable to the caller's groups.
|
||||
func TestSelectPolicy_AllowlistDoesNotLeakAcrossGroups(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
mgr, mockStore := newSelectorMgr(t, ctrl)
|
||||
|
||||
polA := guardedPolicy("pol-a", "acc-1", []string{"grp-a"}, "prov-1", "g-a")
|
||||
polB := guardedPolicy("pol-b", "acc-1", []string{"grp-b"}, "prov-1", "g-b")
|
||||
expectPolicies(mockStore, "acc-1", polA, polB)
|
||||
expectGuardrails(mockStore, "acc-1",
|
||||
allowlistGuardrail("g-a", "acc-1", "gpt-4o"),
|
||||
allowlistGuardrail("g-b", "acc-1", "claude-opus-4"),
|
||||
)
|
||||
|
||||
res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{
|
||||
AccountID: "acc-1",
|
||||
GroupIDs: []string{"grp-a"},
|
||||
ProviderID: "prov-1",
|
||||
Model: "claude-opus-4", // only allowed for grp-b
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.False(t, res.Allow, "grp-b's allowlisted model must not leak to a grp-a caller")
|
||||
assert.Equal(t, denyCodeModelBlocked, res.DenyCode)
|
||||
}
|
||||
|
||||
// TestSelectPolicy_UndeterminedModelFailsClosed proves the fail-closed contract
|
||||
// mirrors the proxy: with a restricted applicable policy and an empty model
|
||||
// (e.g. a path-routed shape the parser couldn't map), the request is denied.
|
||||
func TestSelectPolicy_UndeterminedModelFailsClosed(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
mgr, mockStore := newSelectorMgr(t, ctrl)
|
||||
|
||||
policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1")
|
||||
expectPolicies(mockStore, "acc-1", policy)
|
||||
expectGuardrails(mockStore, "acc-1", allowlistGuardrail("g-1", "acc-1", "gpt-4o"))
|
||||
|
||||
res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{
|
||||
AccountID: "acc-1",
|
||||
GroupIDs: []string{"grp-eng"},
|
||||
ProviderID: "prov-1",
|
||||
Model: "", // undetermined
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.False(t, res.Allow, "an undetermined model must fail closed against a restricted policy")
|
||||
assert.Equal(t, denyCodeModelBlocked, res.DenyCode)
|
||||
}
|
||||
|
||||
// TestSelectPolicy_DisabledAllowlistDoesNotRestrict proves a guardrail whose
|
||||
// model allowlist is disabled imposes no model restriction, even though the
|
||||
// policy references it.
|
||||
func TestSelectPolicy_DisabledAllowlistDoesNotRestrict(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
mgr, mockStore := newSelectorMgr(t, ctrl)
|
||||
|
||||
policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1")
|
||||
disabled := &types.Guardrail{
|
||||
ID: "g-1",
|
||||
AccountID: "acc-1",
|
||||
Checks: types.GuardrailChecks{
|
||||
ModelAllowlist: types.GuardrailModelAllowlist{Enabled: false, Models: []string{"gpt-4o"}},
|
||||
},
|
||||
}
|
||||
expectPolicies(mockStore, "acc-1", policy)
|
||||
expectGuardrails(mockStore, "acc-1", disabled)
|
||||
expectConsumptionBatch(mockStore, nil)
|
||||
|
||||
res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{
|
||||
AccountID: "acc-1",
|
||||
GroupIDs: []string{"grp-eng"},
|
||||
ProviderID: "prov-1",
|
||||
Model: "anything-goes",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, res.Allow, "a disabled allowlist must not restrict the model")
|
||||
assert.Equal(t, "pol-A", res.SelectedPolicyID)
|
||||
}
|
||||
|
||||
// TestSelectPolicy_UnionAcrossPolicyGuardrails proves a policy with multiple
|
||||
// allowlist guardrails permits the union of their models (not just the first).
|
||||
func TestSelectPolicy_UnionAcrossPolicyGuardrails(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
mgr, mockStore := newSelectorMgr(t, ctrl)
|
||||
|
||||
policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1", "g-2")
|
||||
expectPolicies(mockStore, "acc-1", policy)
|
||||
expectGuardrails(mockStore, "acc-1",
|
||||
allowlistGuardrail("g-1", "acc-1", "gpt-4o"),
|
||||
allowlistGuardrail("g-2", "acc-1", "claude-opus-4"),
|
||||
)
|
||||
expectConsumptionBatch(mockStore, nil)
|
||||
|
||||
res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{
|
||||
AccountID: "acc-1",
|
||||
GroupIDs: []string{"grp-eng"},
|
||||
ProviderID: "prov-1",
|
||||
Model: "claude-opus-4", // only in the second guardrail's list
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, res.Allow, "a model in any of the policy's allowlist guardrails must be permitted")
|
||||
assert.Equal(t, "pol-A", res.SelectedPolicyID)
|
||||
}
|
||||
|
||||
// TestSelectPolicy_GuardrailLookupErrorPropagates proves a store failure while
|
||||
// resolving the candidate policies' guardrails surfaces as an error, not a
|
||||
// silent allow/deny.
|
||||
func TestSelectPolicy_GuardrailLookupErrorPropagates(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
mgr, mockStore := newSelectorMgr(t, ctrl)
|
||||
|
||||
policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-1")
|
||||
expectPolicies(mockStore, "acc-1", policy)
|
||||
mockStore.EXPECT().
|
||||
GetAccountAgentNetworkGuardrails(gomock.Any(), gomock.Any(), "acc-1").
|
||||
Return(nil, errors.New("store unavailable"))
|
||||
|
||||
res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{
|
||||
AccountID: "acc-1",
|
||||
GroupIDs: []string{"grp-eng"},
|
||||
ProviderID: "prov-1",
|
||||
Model: "gpt-4o",
|
||||
})
|
||||
require.Error(t, err, "a guardrail-lookup failure must surface as an error")
|
||||
assert.Nil(t, res)
|
||||
}
|
||||
|
||||
// TestSelectPolicy_MissingGuardrailReferenceTreatedAsUnrestricted proves a
|
||||
// policy referencing a guardrail ID absent from the account's set (a stale
|
||||
// reference) imposes no model restriction — same as no guardrail.
|
||||
func TestSelectPolicy_MissingGuardrailReferenceTreatedAsUnrestricted(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
mgr, mockStore := newSelectorMgr(t, ctrl)
|
||||
|
||||
policy := guardedPolicy("pol-A", "acc-1", []string{"grp-eng"}, "prov-1", "g-missing")
|
||||
expectPolicies(mockStore, "acc-1", policy)
|
||||
expectGuardrails(mockStore, "acc-1")
|
||||
expectConsumptionBatch(mockStore, nil)
|
||||
|
||||
res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{
|
||||
AccountID: "acc-1",
|
||||
GroupIDs: []string{"grp-eng"},
|
||||
ProviderID: "prov-1",
|
||||
Model: "anything-goes",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, res.Allow, "an orphaned guardrail reference must not restrict the model")
|
||||
assert.Equal(t, "pol-A", res.SelectedPolicyID)
|
||||
}
|
||||
|
||||
// TestSelectPolicy_PartialCandidatesPermittedAfterModelFilter proves the model
|
||||
// gate narrows candidates before cap scoring: the permitting policy is selected
|
||||
// even though the blocked one has a larger, more attractive cap.
|
||||
func TestSelectPolicy_PartialCandidatesPermittedAfterModelFilter(t *testing.T) {
|
||||
ctrl := gomock.NewController(t)
|
||||
mgr, mockStore := newSelectorMgr(t, ctrl)
|
||||
|
||||
polBig := guardedPolicy("pol-big", "acc-1", []string{"grp-eng"}, "prov-1", "g-restrict")
|
||||
polBig.Limits = types.PolicyLimits{
|
||||
TokenLimit: types.PolicyTokenLimit{Enabled: true, GroupCap: 1_000_000, WindowSeconds: 3600},
|
||||
}
|
||||
polSmall := guardedPolicy("pol-small", "acc-1", []string{"grp-eng"}, "prov-1", "g-permit")
|
||||
polSmall.Limits = types.PolicyLimits{
|
||||
TokenLimit: types.PolicyTokenLimit{Enabled: true, GroupCap: 100, WindowSeconds: 3600},
|
||||
}
|
||||
expectPolicies(mockStore, "acc-1", polBig, polSmall)
|
||||
expectGuardrails(mockStore, "acc-1",
|
||||
allowlistGuardrail("g-restrict", "acc-1", "gpt-4o"),
|
||||
allowlistGuardrail("g-permit", "acc-1", "claude-opus-4"),
|
||||
)
|
||||
expectConsumptionBatch(mockStore, nil)
|
||||
|
||||
res, err := mgr.SelectPolicyForRequest(context.Background(), PolicySelectionInput{
|
||||
AccountID: "acc-1",
|
||||
GroupIDs: []string{"grp-eng"},
|
||||
ProviderID: "prov-1",
|
||||
Model: "claude-opus-4", // only pol-small's guardrail permits this
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, res.Allow)
|
||||
assert.Equal(t, "pol-small", res.SelectedPolicyID,
|
||||
"the model filter must exclude pol-big before cap scoring")
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/catalog"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
|
||||
)
|
||||
|
||||
func TestPrepareProviderAPIKey(t *testing.T) {
|
||||
t.Run("required create rejects an empty key", func(t *testing.T) {
|
||||
provider := &types.Provider{ProviderID: "openai_api"}
|
||||
err := prepareProviderAPIKey(provider, nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "api_key is required")
|
||||
})
|
||||
|
||||
t.Run("Ollama Cloud requires a key", func(t *testing.T) {
|
||||
provider := &types.Provider{ProviderID: "ollama_cloud"}
|
||||
err := prepareProviderAPIKey(provider, nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "api_key is required")
|
||||
})
|
||||
|
||||
t.Run("Ollama Cloud accepts a key", func(t *testing.T) {
|
||||
provider := &types.Provider{
|
||||
ProviderID: "ollama_cloud",
|
||||
APIKey: "ollama-cloud-key",
|
||||
APIKeyProvided: true,
|
||||
}
|
||||
require.NoError(t, prepareProviderAPIKey(provider, nil))
|
||||
assert.Equal(t, "ollama-cloud-key", provider.APIKey)
|
||||
})
|
||||
|
||||
t.Run("optional create accepts an empty key", func(t *testing.T) {
|
||||
provider := &types.Provider{ProviderID: "ollama"}
|
||||
require.NoError(t, prepareProviderAPIKey(provider, nil))
|
||||
assert.Empty(t, provider.APIKey)
|
||||
})
|
||||
|
||||
t.Run("optional update omission preserves the existing key", func(t *testing.T) {
|
||||
existing := &types.Provider{ProviderID: "ollama", APIKey: "existing"}
|
||||
provider := &types.Provider{ProviderID: "ollama"}
|
||||
require.NoError(t, prepareProviderAPIKey(provider, existing))
|
||||
assert.Equal(t, "existing", provider.APIKey)
|
||||
})
|
||||
|
||||
t.Run("optional update explicit empty clears the existing key", func(t *testing.T) {
|
||||
existing := &types.Provider{ProviderID: "ollama", APIKey: "existing"}
|
||||
provider := &types.Provider{ProviderID: "ollama", APIKeyProvided: true}
|
||||
require.NoError(t, prepareProviderAPIKey(provider, existing))
|
||||
assert.Empty(t, provider.APIKey)
|
||||
})
|
||||
|
||||
t.Run("changing to optional without a key does not carry the old secret", func(t *testing.T) {
|
||||
existing := &types.Provider{ProviderID: "openai_api", APIKey: "sk-old-provider"}
|
||||
provider := &types.Provider{ProviderID: "ollama"}
|
||||
require.NoError(t, prepareProviderAPIKey(provider, existing))
|
||||
assert.Empty(t, provider.APIKey)
|
||||
})
|
||||
|
||||
t.Run("changing to required without a key fails", func(t *testing.T) {
|
||||
existing := &types.Provider{ProviderID: "ollama", APIKey: "old-optional-key"}
|
||||
provider := &types.Provider{ProviderID: "openai_api"}
|
||||
err := prepareProviderAPIKey(provider, existing)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "api_key is required")
|
||||
assert.Empty(t, provider.APIKey)
|
||||
})
|
||||
|
||||
t.Run("required update omission preserves the existing key", func(t *testing.T) {
|
||||
existing := &types.Provider{ProviderID: "openai_api", APIKey: "sk-existing"}
|
||||
provider := &types.Provider{ProviderID: "openai_api"}
|
||||
require.NoError(t, prepareProviderAPIKey(provider, existing))
|
||||
assert.Equal(t, "sk-existing", provider.APIKey)
|
||||
})
|
||||
|
||||
t.Run("required update rejects a preserved whitespace-only key", func(t *testing.T) {
|
||||
existing := &types.Provider{ProviderID: "openai_api", APIKey: " "}
|
||||
provider := &types.Provider{ProviderID: "openai_api"}
|
||||
err := prepareProviderAPIKey(provider, existing)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "api_key is required")
|
||||
assert.Empty(t, provider.APIKey)
|
||||
})
|
||||
|
||||
t.Run("required update explicit empty is rejected", func(t *testing.T) {
|
||||
existing := &types.Provider{ProviderID: "openai_api", APIKey: "sk-existing"}
|
||||
provider := &types.Provider{ProviderID: "openai_api", APIKeyProvided: true}
|
||||
err := prepareProviderAPIKey(provider, existing)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "api_key is required")
|
||||
})
|
||||
|
||||
t.Run("unknown catalog provider is rejected", func(t *testing.T) {
|
||||
provider := &types.Provider{ProviderID: "unknown", APIKey: "secret"}
|
||||
err := prepareProviderAPIKey(provider, nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "not a known catalog provider")
|
||||
})
|
||||
|
||||
t.Run("none mode clears a stale key", func(t *testing.T) {
|
||||
entry := catalog.Provider{ID: "authless", AuthMode: catalog.AuthModeNone}
|
||||
existing := &types.Provider{ProviderID: "authless", APIKey: "stale-secret"}
|
||||
provider := &types.Provider{ProviderID: "authless"}
|
||||
require.NoError(t, prepareProviderAPIKeyForEntry(provider, existing, entry))
|
||||
assert.Empty(t, provider.APIKey)
|
||||
})
|
||||
|
||||
t.Run("none mode rejects a supplied key", func(t *testing.T) {
|
||||
entry := catalog.Provider{ID: "authless", AuthMode: catalog.AuthModeNone}
|
||||
provider := &types.Provider{
|
||||
ProviderID: "authless",
|
||||
APIKey: "unexpected-secret",
|
||||
APIKeyProvided: true,
|
||||
}
|
||||
err := prepareProviderAPIKeyForEntry(provider, nil, entry)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "not supported")
|
||||
})
|
||||
}
|
||||
@@ -235,7 +235,12 @@ func SynthesizeServices(ctx context.Context, s store.Store, accountID string) ([
|
||||
|
||||
mergedGuardrails := mergeGuardrails(enabledPolicies, guardrailsByID)
|
||||
applyAccountCollectionControls(&mergedGuardrails, settings)
|
||||
guardrailJSON, err := marshalGuardrailConfig(mergedGuardrails)
|
||||
// The proxy guardrail is a per-provider fail-closed backstop; the
|
||||
// authoritative per-policy/group decision is management's
|
||||
// SelectPolicyForRequest. A provider lands in this map only when every
|
||||
// authorising policy restricts models.
|
||||
providerAllowlists := buildProviderAllowlists(enabledPolicies, guardrailsByID)
|
||||
guardrailJSON, err := marshalGuardrailConfig(providerAllowlists, mergedGuardrails.PromptCapture)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -780,10 +785,12 @@ func buildMiddlewareChain(routerCfgJSON, identityInjectJSON, guardrailJSON []byt
|
||||
|
||||
// guardrailConfig is the JSON shape the proxy-side llm_guardrail
|
||||
// middleware expects. Mirrors the proxy registration documented in
|
||||
// the management→proxy contract.
|
||||
// the management→proxy contract. provider_allowlists is keyed by the
|
||||
// resolved provider id llm_router stamps; a provider absent from the map is
|
||||
// unrestricted at the proxy layer.
|
||||
type guardrailConfig struct {
|
||||
ModelAllowlist []string `json:"model_allowlist,omitempty"`
|
||||
PromptCapture guardrailPromptCapture `json:"prompt_capture"`
|
||||
ProviderAllowlists map[string][]string `json:"provider_allowlists,omitempty"`
|
||||
PromptCapture guardrailPromptCapture `json:"prompt_capture"`
|
||||
}
|
||||
|
||||
type guardrailPromptCapture struct {
|
||||
@@ -828,13 +835,10 @@ func applyAccountCollectionControls(merged *MergedGuardrails, settings *types.Se
|
||||
merged.PromptCapture.RedactPii = settings.RedactPii || merged.PromptCapture.RedactPii
|
||||
}
|
||||
|
||||
func marshalGuardrailConfig(merged MergedGuardrails) ([]byte, error) {
|
||||
func marshalGuardrailConfig(providerAllowlists map[string][]string, capture MergedPromptCapture) ([]byte, error) {
|
||||
cfg := guardrailConfig{
|
||||
ModelAllowlist: merged.ModelAllowlist,
|
||||
PromptCapture: guardrailPromptCapture{
|
||||
Enabled: merged.PromptCapture.Enabled,
|
||||
RedactPii: merged.PromptCapture.RedactPii,
|
||||
},
|
||||
ProviderAllowlists: providerAllowlists,
|
||||
PromptCapture: guardrailPromptCapture(capture),
|
||||
}
|
||||
out, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
@@ -843,6 +847,74 @@ func marshalGuardrailConfig(merged MergedGuardrails) ([]byte, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// buildProviderAllowlists returns the proxy's per-provider backstop: a provider
|
||||
// is included only when every authorising policy restricts models (their union);
|
||||
// if any leaves it unrestricted it is omitted, so management decides per group.
|
||||
func buildProviderAllowlists(policies []*types.Policy, byID map[string]*types.Guardrail) map[string][]string {
|
||||
type providerAcc struct {
|
||||
models map[string]struct{}
|
||||
anyUnrestricted bool
|
||||
}
|
||||
accs := make(map[string]*providerAcc)
|
||||
for _, p := range policies {
|
||||
if p == nil {
|
||||
continue
|
||||
}
|
||||
restricted, models := policyModelAllowlist(p, byID)
|
||||
for _, providerID := range p.DestinationProviderIDs {
|
||||
if providerID == "" {
|
||||
continue
|
||||
}
|
||||
acc, ok := accs[providerID]
|
||||
if !ok {
|
||||
acc = &providerAcc{models: make(map[string]struct{})}
|
||||
accs[providerID] = acc
|
||||
}
|
||||
if !restricted {
|
||||
acc.anyUnrestricted = true
|
||||
continue
|
||||
}
|
||||
for _, m := range models {
|
||||
acc.models[m] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make(map[string][]string, len(accs))
|
||||
for providerID, acc := range accs {
|
||||
if acc.anyUnrestricted {
|
||||
continue
|
||||
}
|
||||
models := make([]string, 0, len(acc.models))
|
||||
for m := range acc.models {
|
||||
models = append(models, m)
|
||||
}
|
||||
sort.Strings(models)
|
||||
out[providerID] = models
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// policyModelAllowlist reports whether a policy restricts models (has an
|
||||
// allowlist-enabled guardrail) and the union of allowed models. Models are
|
||||
// verbatim; the proxy factory lowercases/trims them at decode time.
|
||||
func policyModelAllowlist(p *types.Policy, byID map[string]*types.Guardrail) (bool, []string) {
|
||||
restricted := false
|
||||
var models []string
|
||||
for _, gID := range p.GuardrailIDs {
|
||||
g, ok := byID[gID]
|
||||
if !ok || g == nil || !g.Checks.ModelAllowlist.Enabled {
|
||||
continue
|
||||
}
|
||||
restricted = true
|
||||
for _, m := range g.Checks.ModelAllowlist.Models {
|
||||
if m != "" {
|
||||
models = append(models, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
return restricted, models
|
||||
}
|
||||
|
||||
// buildAccountService composes the per-account gateway Service. The
|
||||
// target carries the noop placeholder URL — the router middleware
|
||||
// rewrites every request to the matched provider's upstream before the
|
||||
@@ -906,33 +978,23 @@ const (
|
||||
noopUpstreamPort = uint16(443)
|
||||
)
|
||||
|
||||
// providerAuthHeader builds the upstream auth header pair for a provider from
|
||||
// its catalog entry. Optional providers with no key and providers using no
|
||||
// authentication return an empty pair. Otherwise, the synthesiser substitutes
|
||||
// the decrypted API key into the catalog template and returns the pair the
|
||||
// router injects after stripping inbound vendor credentials.
|
||||
// providerAuthHeader builds the upstream auth header pair for a
|
||||
// provider from its catalog entry. The catalog declares which header
|
||||
// name and template a provider's API expects; the synthesiser
|
||||
// substitutes the provider's decrypted API key into the template and
|
||||
// returns the (name, value) pair the router middleware injects after
|
||||
// stripping the inbound vendor auth headers.
|
||||
func providerAuthHeader(p *types.Provider) (name, value, gcpSAKeyB64 string, err error) {
|
||||
entry, ok := catalog.Lookup(p.ProviderID)
|
||||
if !ok {
|
||||
return "", "", "", fmt.Errorf("provider %s references unknown catalog id %q", p.ID, p.ProviderID)
|
||||
}
|
||||
switch entry.EffectiveAuthMode() {
|
||||
case catalog.AuthModeNone:
|
||||
return "", "", "", nil
|
||||
case catalog.AuthModeOptional:
|
||||
if strings.TrimSpace(p.APIKey) == "" {
|
||||
return "", "", "", nil
|
||||
}
|
||||
case catalog.AuthModeRequired:
|
||||
if strings.TrimSpace(p.APIKey) == "" {
|
||||
return "", "", "", fmt.Errorf("provider %s has no api key", p.ID)
|
||||
}
|
||||
default:
|
||||
return "", "", "", fmt.Errorf("catalog entry %q has invalid authentication mode %q", p.ProviderID, entry.AuthMode)
|
||||
}
|
||||
if entry.AuthHeaderName == "" || entry.AuthHeaderTemplate == "" {
|
||||
return "", "", "", fmt.Errorf("catalog entry %q has no auth header configured", p.ProviderID)
|
||||
}
|
||||
if p.APIKey == "" {
|
||||
return "", "", "", fmt.Errorf("provider %s has no api key", p.ID)
|
||||
}
|
||||
// A "keyfile::<base64 json>" api_key is a GCP service-account key, not a
|
||||
// static bearer. The proxy mints + refreshes a short-lived OAuth token from
|
||||
// it at request time, so carry the key material on the route and emit no
|
||||
@@ -996,38 +1058,11 @@ func unionSourceGroups(policies []*types.Policy) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// MergedGuardrails is the JSON shape passed to the proxy via the
|
||||
// guardrail middleware's config_json. Mirrors the proxy-side
|
||||
// expectations and is intentionally distinct from
|
||||
// types.GuardrailChecks so we can evolve either side independently.
|
||||
// MergedGuardrails is the synthesiser's fold target. Only prompt capture is
|
||||
// merged here — the model allowlist is emitted per-provider, and
|
||||
// token/budget/retention moved onto Policy.Limits and account Settings.
|
||||
type MergedGuardrails struct {
|
||||
ModelAllowlist []string `json:"model_allowlist,omitempty"`
|
||||
TokenLimits MergedTokenLimits `json:"token_limits"`
|
||||
Budget MergedBudget `json:"budget"`
|
||||
PromptCapture MergedPromptCapture `json:"prompt_capture"`
|
||||
Retention MergedRetention `json:"retention"`
|
||||
}
|
||||
|
||||
type MergedTokenLimits struct {
|
||||
Hourly *MergedTokenWindow `json:"hourly,omitempty"`
|
||||
Daily *MergedTokenWindow `json:"daily,omitempty"`
|
||||
Monthly *MergedTokenWindow `json:"monthly,omitempty"`
|
||||
}
|
||||
|
||||
type MergedTokenWindow struct {
|
||||
MaxInputTokens int `json:"max_input_tokens,omitempty"`
|
||||
MaxOutputTokens int `json:"max_output_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type MergedBudget struct {
|
||||
Hourly *MergedBudgetWindow `json:"hourly,omitempty"`
|
||||
Daily *MergedBudgetWindow `json:"daily,omitempty"`
|
||||
Monthly *MergedBudgetWindow `json:"monthly,omitempty"`
|
||||
}
|
||||
|
||||
type MergedBudgetWindow struct {
|
||||
SoftCapUSD float64 `json:"soft_cap_usd,omitempty"`
|
||||
HardCapUSD float64 `json:"hard_cap_usd,omitempty"`
|
||||
PromptCapture MergedPromptCapture
|
||||
}
|
||||
|
||||
type MergedPromptCapture struct {
|
||||
@@ -1035,64 +1070,31 @@ type MergedPromptCapture struct {
|
||||
RedactPii bool `json:"redact_pii"`
|
||||
}
|
||||
|
||||
type MergedRetention struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Days int `json:"days"`
|
||||
}
|
||||
|
||||
// mergeGuardrails computes the effective guardrail spec applied at the
|
||||
// proxy, given the referencing policies and the account's guardrail
|
||||
// catalogue. Policy enabled-ness is the caller's responsibility — only
|
||||
// enabled policies should be passed in.
|
||||
// mergeGuardrails folds the referencing policies' guardrails into the
|
||||
// prompt-capture decision only. The model allowlist is enforced per-policy/group
|
||||
// in management and shipped per-provider; token/budget/retention live off
|
||||
// guardrails now.
|
||||
//
|
||||
// Merge rules:
|
||||
// - Model allowlist: union of allowlists across policies that enable it.
|
||||
// - Token / Budget: most-restrictive (min of non-zero caps) per window.
|
||||
// - Prompt capture: enabled if any policy enables it; redact_pii sticks
|
||||
// if any enabling policy turns it on.
|
||||
// - Retention: enabled if any enables it; smallest non-zero days wins.
|
||||
// Merge rule — prompt capture: enabled if any policy enables it; redact_pii
|
||||
// sticks if any enabling policy turns it on.
|
||||
func mergeGuardrails(policies []*types.Policy, byID map[string]*types.Guardrail) MergedGuardrails {
|
||||
merged := MergedGuardrails{}
|
||||
allowlist := make(map[string]struct{})
|
||||
allowlistEnabled := false
|
||||
|
||||
for _, policy := range policies {
|
||||
for _, gID := range policy.GuardrailIDs {
|
||||
g, ok := byID[gID]
|
||||
if !ok || g == nil {
|
||||
continue
|
||||
}
|
||||
mergeGuardrail(g, &merged, allowlist, &allowlistEnabled)
|
||||
mergeGuardrail(g, &merged)
|
||||
}
|
||||
}
|
||||
|
||||
if allowlistEnabled {
|
||||
merged.ModelAllowlist = make([]string, 0, len(allowlist))
|
||||
for m := range allowlist {
|
||||
merged.ModelAllowlist = append(merged.ModelAllowlist, m)
|
||||
}
|
||||
sort.Strings(merged.ModelAllowlist)
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
// mergeGuardrail folds a single guardrail's enabled checks into the
|
||||
// running merge: model-allowlist models join the shared set (and flip
|
||||
// allowlistEnabled), and prompt-capture / redact-pii stick once any
|
||||
// enabling guardrail turns them on.
|
||||
//
|
||||
// TokenLimits, Budget, and Retention have moved off guardrails — token
|
||||
// and budget caps now live on the Policy itself (Policy.Limits) and
|
||||
// retention moves to account-level Settings — so they are not merged here.
|
||||
func mergeGuardrail(g *types.Guardrail, merged *MergedGuardrails, allowlist map[string]struct{}, allowlistEnabled *bool) {
|
||||
if g.Checks.ModelAllowlist.Enabled {
|
||||
*allowlistEnabled = true
|
||||
for _, m := range g.Checks.ModelAllowlist.Models {
|
||||
if m != "" {
|
||||
allowlist[m] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
// mergeGuardrail folds a single guardrail's prompt-capture settings into the
|
||||
// running merge: enabled / redact-pii stick once any enabling guardrail turns
|
||||
// them on.
|
||||
func mergeGuardrail(g *types.Guardrail, merged *MergedGuardrails) {
|
||||
if g.Checks.PromptCapture.Enabled {
|
||||
merged.PromptCapture.Enabled = true
|
||||
if g.Checks.PromptCapture.RedactPii {
|
||||
|
||||
@@ -81,8 +81,8 @@ func TestSynthesizeServices_RealStore_PromptCaptureAccountIsSoleControl(t *testi
|
||||
require.Len(t, services, 1)
|
||||
|
||||
cfg := decodeServiceGuardrailConfig(t, services[0])
|
||||
assert.Equal(t, []string{"gpt-5.4"}, cfg.ModelAllowlist,
|
||||
"model allowlist is a pure policy guardrail and must always reach the config")
|
||||
assert.Equal(t, map[string][]string{"prov-1": {"gpt-5.4"}}, cfg.ProviderAllowlists,
|
||||
"model allowlist is a pure policy guardrail and must reach the per-provider config")
|
||||
assert.False(t, cfg.PromptCapture.Enabled,
|
||||
"prompt capture must be off when the account toggle is off, even with a capture-enabled guardrail")
|
||||
}
|
||||
@@ -172,7 +172,7 @@ func TestSynthesizeServices_RealStore_NoGuardrail_CaptureOff(t *testing.T) {
|
||||
require.Len(t, services, 1, "exactly one synth service expected")
|
||||
|
||||
cfg := decodeServiceGuardrailConfig(t, services[0])
|
||||
assert.Empty(t, cfg.ModelAllowlist, "no guardrail → no allowlist")
|
||||
assert.Empty(t, cfg.ProviderAllowlists, "no guardrail → provider unrestricted (absent from map)")
|
||||
assert.False(t, cfg.PromptCapture.Enabled, "no guardrail → prompt capture off by default")
|
||||
assert.False(t, cfg.PromptCapture.RedactPii, "no guardrail → redact off by default")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package agentnetwork
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
|
||||
)
|
||||
|
||||
// policyForProviders builds an enabled policy authorising the given providers
|
||||
// under the given guardrails (both optional). Groups are irrelevant to
|
||||
// buildProviderAllowlists, which keys purely on destination provider.
|
||||
func policyForProviders(id string, guardrailIDs []string, providerIDs ...string) *types.Policy {
|
||||
return &types.Policy{
|
||||
ID: id,
|
||||
Enabled: true,
|
||||
DestinationProviderIDs: providerIDs,
|
||||
GuardrailIDs: guardrailIDs,
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildProviderAllowlists(t *testing.T) {
|
||||
byID := map[string]*types.Guardrail{
|
||||
"g-4o": allowlistGuardrail("g-4o", "acc-1", "gpt-4o"),
|
||||
"g-opus": allowlistGuardrail("g-opus", "acc-1", "claude-opus-4"),
|
||||
"g-disabled": {ID: "g-disabled", Checks: types.GuardrailChecks{ModelAllowlist: types.GuardrailModelAllowlist{Enabled: false, Models: []string{"gpt-4o"}}}},
|
||||
}
|
||||
|
||||
t.Run("all authorising policies restrict yields per-provider union", func(t *testing.T) {
|
||||
policies := []*types.Policy{
|
||||
policyForProviders("p1", []string{"g-4o"}, "prov-x"),
|
||||
policyForProviders("p2", []string{"g-opus"}, "prov-x"),
|
||||
}
|
||||
got := buildProviderAllowlists(policies, byID)
|
||||
assert.Equal(t, map[string][]string{"prov-x": {"claude-opus-4", "gpt-4o"}}, got,
|
||||
"a provider every policy restricts carries the sorted union of their models")
|
||||
})
|
||||
|
||||
t.Run("any un-guardrailed policy leaves the provider unrestricted (omitted)", func(t *testing.T) {
|
||||
policies := []*types.Policy{
|
||||
policyForProviders("p1", []string{"g-4o"}, "prov-x"),
|
||||
policyForProviders("p2", nil, "prov-x"), // no guardrail
|
||||
}
|
||||
got := buildProviderAllowlists(policies, byID)
|
||||
assert.NotContains(t, got, "prov-x",
|
||||
"a provider reachable by an un-guardrailed policy must be omitted so the proxy treats it as unrestricted")
|
||||
})
|
||||
|
||||
t.Run("a disabled allowlist counts as unrestricted", func(t *testing.T) {
|
||||
policies := []*types.Policy{
|
||||
policyForProviders("p1", []string{"g-disabled"}, "prov-x"),
|
||||
}
|
||||
got := buildProviderAllowlists(policies, byID)
|
||||
assert.NotContains(t, got, "prov-x",
|
||||
"a policy whose only guardrail has a disabled allowlist is unrestricted")
|
||||
})
|
||||
|
||||
t.Run("providers are isolated from one another", func(t *testing.T) {
|
||||
policies := []*types.Policy{
|
||||
policyForProviders("p1", []string{"g-4o"}, "prov-x"),
|
||||
policyForProviders("p2", []string{"g-opus"}, "prov-y"),
|
||||
}
|
||||
got := buildProviderAllowlists(policies, byID)
|
||||
assert.Equal(t, []string{"gpt-4o"}, got["prov-x"], "prov-x keeps only its own model")
|
||||
assert.Equal(t, []string{"claude-opus-4"}, got["prov-y"], "prov-y keeps only its own model")
|
||||
})
|
||||
|
||||
t.Run("one policy authorising two providers restricts both", func(t *testing.T) {
|
||||
policies := []*types.Policy{
|
||||
policyForProviders("p1", []string{"g-4o"}, "prov-x", "prov-y"),
|
||||
}
|
||||
got := buildProviderAllowlists(policies, byID)
|
||||
assert.Equal(t, []string{"gpt-4o"}, got["prov-x"])
|
||||
assert.Equal(t, []string{"gpt-4o"}, got["prov-y"])
|
||||
})
|
||||
|
||||
t.Run("union across a single policy's guardrails", func(t *testing.T) {
|
||||
policies := []*types.Policy{
|
||||
policyForProviders("p1", []string{"g-4o", "g-opus"}, "prov-x"),
|
||||
}
|
||||
got := buildProviderAllowlists(policies, byID)
|
||||
assert.ElementsMatch(t, []string{"claude-opus-4", "gpt-4o"}, got["prov-x"],
|
||||
"a policy's own multiple allowlist guardrails union together")
|
||||
})
|
||||
|
||||
t.Run("an enabled allowlist with no models denies everything", func(t *testing.T) {
|
||||
empty := map[string]*types.Guardrail{"g-empty": allowlistGuardrail("g-empty", "acc-1")}
|
||||
got := buildProviderAllowlists([]*types.Policy{
|
||||
policyForProviders("p1", []string{"g-empty"}, "prov-x"),
|
||||
}, empty)
|
||||
assert.Equal(t, map[string][]string{"prov-x": {}}, got,
|
||||
"an enabled-but-empty allowlist is restricted with an empty set, not unrestricted")
|
||||
})
|
||||
}
|
||||
@@ -1031,8 +1031,12 @@ func TestSynthesizeServices_GuardrailMerge_AllowlistUnion_LimitsRestrictive(t *t
|
||||
|
||||
var cfg guardrailConfig
|
||||
require.NoError(t, json.Unmarshal(guardrailJSON, &cfg), "guardrail config must unmarshal cleanly")
|
||||
assert.ElementsMatch(t, []string{"gpt-5.4-mini", "gpt-5.4-pro"}, cfg.ModelAllowlist,
|
||||
"model allowlist union must keep both models")
|
||||
// Both policies restrict the same provider, so the per-provider backstop
|
||||
// carries the union of their models — a coarse gate that management's
|
||||
// per-policy/group check narrows; it only blocks models outside the union
|
||||
// when management is down.
|
||||
assert.ElementsMatch(t, []string{"gpt-5.4-mini", "gpt-5.4-pro"}, cfg.ProviderAllowlists["prov-1"],
|
||||
"per-provider allowlist union must keep both models")
|
||||
}
|
||||
|
||||
func TestSynthesizeServices_BackfillsMissingSessionKeys(t *testing.T) {
|
||||
@@ -1219,102 +1223,3 @@ func TestSynthesizeServices_EmptyAPIKey_FailsClosed(t *testing.T) {
|
||||
require.Error(t, err, "synthesis must refuse a provider with no api key")
|
||||
assert.Contains(t, err.Error(), "no api key", "error must surface the missing credential")
|
||||
}
|
||||
|
||||
func TestSynthesizeServices_OllamaOptionalAPIKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
apiKey string
|
||||
wantHeaderName string
|
||||
wantHeaderValue string
|
||||
}{
|
||||
{
|
||||
name: "no key emits no replacement auth header",
|
||||
},
|
||||
{
|
||||
name: "configured key emits bearer auth",
|
||||
apiKey: "protected-endpoint-token",
|
||||
wantHeaderName: "Authorization",
|
||||
wantHeaderValue: "Bearer protected-endpoint-token",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctrl := gomock.NewController(t)
|
||||
defer ctrl.Finish()
|
||||
mockStore := store.NewMockStore(ctrl)
|
||||
|
||||
provider := newSynthTestProvider()
|
||||
provider.ProviderID = "ollama"
|
||||
provider.Name = "Ollama"
|
||||
provider.UpstreamURL = "http://ollama.internal:11434"
|
||||
provider.APIKey = tt.apiKey
|
||||
provider.Models = []types.ProviderModel{}
|
||||
policy := newSynthTestPolicy(provider.ID, "grp-eng", "")
|
||||
|
||||
expectSynthBaseInputs(mockStore, ctx, newSynthTestSettings(),
|
||||
[]*types.Provider{provider},
|
||||
[]*types.Policy{policy},
|
||||
[]*types.Guardrail{})
|
||||
|
||||
services, err := SynthesizeServices(ctx, mockStore, testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, services, 1)
|
||||
|
||||
var routerCfg routerConfig
|
||||
for _, middleware := range services[0].Targets[0].Options.Middlewares {
|
||||
if middleware.ID == middlewareIDLLMRouter {
|
||||
require.NoError(t, json.Unmarshal(middleware.ConfigJSON, &routerCfg))
|
||||
break
|
||||
}
|
||||
}
|
||||
require.Len(t, routerCfg.Providers, 1)
|
||||
assert.Equal(t, tt.wantHeaderName, routerCfg.Providers[0].AuthHeaderName)
|
||||
assert.Equal(t, tt.wantHeaderValue, routerCfg.Providers[0].AuthHeaderValue)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSynthesizeServices_OllamaCloudRoute(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctrl := gomock.NewController(t)
|
||||
defer ctrl.Finish()
|
||||
mockStore := store.NewMockStore(ctrl)
|
||||
|
||||
provider := newSynthTestProvider()
|
||||
provider.ProviderID = "ollama_cloud"
|
||||
provider.Name = "Ollama Cloud"
|
||||
provider.UpstreamURL = "https://ollama.com"
|
||||
provider.APIKey = "ollama-cloud-key"
|
||||
provider.Models = []types.ProviderModel{{ID: "gpt-oss:120b"}}
|
||||
policy := newSynthTestPolicy(provider.ID, "grp-eng", "")
|
||||
|
||||
expectSynthBaseInputs(mockStore, ctx, newSynthTestSettings(),
|
||||
[]*types.Provider{provider},
|
||||
[]*types.Policy{policy},
|
||||
[]*types.Guardrail{})
|
||||
|
||||
services, err := SynthesizeServices(ctx, mockStore, testAccountID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, services, 1)
|
||||
|
||||
var routerCfg routerConfig
|
||||
for _, middleware := range services[0].Targets[0].Options.Middlewares {
|
||||
if middleware.ID == middlewareIDLLMRouter {
|
||||
require.NoError(t, json.Unmarshal(middleware.ConfigJSON, &routerCfg))
|
||||
break
|
||||
}
|
||||
}
|
||||
require.Len(t, routerCfg.Providers, 1)
|
||||
|
||||
route := routerCfg.Providers[0]
|
||||
assert.Empty(t, route.Vendor, "Ollama Cloud preserves the untagged Ollama/vLLM/custom routing behavior")
|
||||
assert.Equal(t, []string{"gpt-oss:120b"}, route.Models)
|
||||
assert.Equal(t, "https", route.UpstreamScheme)
|
||||
assert.Equal(t, "ollama.com", route.UpstreamHost)
|
||||
assert.Empty(t, route.UpstreamPath)
|
||||
assert.Equal(t, "Authorization", route.AuthHeaderName)
|
||||
assert.Equal(t, "Bearer ollama-cloud-key", route.AuthHeaderValue)
|
||||
assert.False(t, route.SkipTLSVerify)
|
||||
}
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
package types
|
||||
|
||||
// DiscoveredModel is a normalized model identifier returned by a provider's
|
||||
// persisted upstream endpoint. Discovery does not persist models; the caller
|
||||
// must explicitly update the provider to opt into the returned list.
|
||||
type DiscoveredModel struct {
|
||||
ID string
|
||||
Label string
|
||||
}
|
||||
|
||||
// ModelDiscoveryResult describes a successful proxy-executed discovery probe.
|
||||
// RequestID correlates the management request with the proxy control message,
|
||||
// and ProxyCluster identifies the network vantage point that ran it.
|
||||
type ModelDiscoveryResult struct {
|
||||
RequestID string
|
||||
Source string
|
||||
ProxyCluster string
|
||||
Models []DiscoveredModel
|
||||
}
|
||||
@@ -33,10 +33,6 @@ type Provider struct {
|
||||
// the operator selected.
|
||||
UpstreamURL string `gorm:"column:upstream_url"`
|
||||
APIKey string `gorm:"column:api_key"`
|
||||
// APIKeyProvided records whether api_key was present in the request. It is
|
||||
// transient and lets updates distinguish omission (preserve) from an
|
||||
// explicit empty value (clear an optional credential).
|
||||
APIKeyProvided bool `gorm:"-" json:"-"`
|
||||
// ExtraValues holds operator-typed values for catalog-declared
|
||||
// ExtraHeaders (see catalog.Provider.ExtraHeaders). Keyed by
|
||||
// header name (e.g. "x-portkey-config"); a non-empty value is
|
||||
@@ -101,16 +97,15 @@ func NewProvider(accountID string) *Provider {
|
||||
}
|
||||
}
|
||||
|
||||
// FromAPIRequest applies the request payload onto the receiver. APIKeyProvided
|
||||
// preserves the distinction between an omitted api_key and an explicit empty
|
||||
// value; the manager applies the catalog-specific preserve/clear semantics.
|
||||
// FromAPIRequest applies the request payload onto the receiver. The api_key
|
||||
// is only overwritten when the caller provided one — empty/nil leaves the
|
||||
// existing key intact, so updates can omit it.
|
||||
func (p *Provider) FromAPIRequest(req *api.AgentNetworkProviderRequest) {
|
||||
p.ProviderID = req.ProviderId
|
||||
p.Name = req.Name
|
||||
p.UpstreamURL = req.UpstreamUrl
|
||||
p.APIKeyProvided = req.ApiKey != nil
|
||||
if req.ApiKey != nil {
|
||||
p.APIKey = strings.TrimSpace(*req.ApiKey)
|
||||
if req.ApiKey != nil && strings.TrimSpace(*req.ApiKey) != "" {
|
||||
p.APIKey = *req.ApiKey
|
||||
}
|
||||
if req.ExtraValues != nil {
|
||||
// Replace the whole map (rather than merge) so unsetting a
|
||||
@@ -183,7 +178,6 @@ func (p *Provider) ToAPIResponse() *api.AgentNetworkProvider {
|
||||
UpstreamUrl: p.UpstreamURL,
|
||||
Models: models,
|
||||
Enabled: p.Enabled,
|
||||
HasApiKey: strings.TrimSpace(p.APIKey) != "",
|
||||
SkipTlsVerification: p.SkipTLSVerification,
|
||||
MetadataDisabled: p.MetadataDisabled,
|
||||
CreatedAt: &created,
|
||||
|
||||
@@ -77,45 +77,3 @@ func TestProvider_MetadataDisabled_RoundTrip(t *testing.T) {
|
||||
assert.False(t, p.MetadataDisabled, "explicit false must clear metadata_disabled")
|
||||
assert.False(t, p.ToAPIResponse().MetadataDisabled, "response must reflect the cleared value")
|
||||
}
|
||||
|
||||
func TestProvider_APIKeyPresenceAndResponse(t *testing.T) {
|
||||
base := func() *api.AgentNetworkProviderRequest {
|
||||
return &api.AgentNetworkProviderRequest{
|
||||
ProviderId: "ollama",
|
||||
Name: "Ollama",
|
||||
UpstreamUrl: "http://ollama.internal:11434",
|
||||
}
|
||||
}
|
||||
|
||||
p := NewProvider("acc-1")
|
||||
p.FromAPIRequest(base())
|
||||
assert.False(t, p.APIKeyProvided, "omission must remain distinguishable from an explicit clear")
|
||||
assert.False(t, p.ToAPIResponse().HasApiKey)
|
||||
|
||||
key := "protected-endpoint-token"
|
||||
withKey := base()
|
||||
withKey.ApiKey = &key
|
||||
p.FromAPIRequest(withKey)
|
||||
assert.True(t, p.APIKeyProvided)
|
||||
assert.Equal(t, key, p.APIKey)
|
||||
assert.True(t, p.ToAPIResponse().HasApiKey)
|
||||
|
||||
paddedKey := " \tprotected-endpoint-token\n"
|
||||
withPaddedKey := base()
|
||||
withPaddedKey.ApiKey = &paddedKey
|
||||
p.FromAPIRequest(withPaddedKey)
|
||||
assert.True(t, p.APIKeyProvided)
|
||||
assert.Equal(t, key, p.APIKey, "non-blank API keys must be normalized before storage")
|
||||
assert.True(t, p.ToAPIResponse().HasApiKey, "the response must reflect the normalized stored key")
|
||||
|
||||
empty := ""
|
||||
clearKey := base()
|
||||
clearKey.ApiKey = &empty
|
||||
p.FromAPIRequest(clearKey)
|
||||
assert.True(t, p.APIKeyProvided, "explicit empty must be carried to the manager as a clear")
|
||||
assert.Empty(t, p.APIKey)
|
||||
assert.False(t, p.ToAPIResponse().HasApiKey)
|
||||
|
||||
p.FromAPIRequest(base())
|
||||
assert.False(t, p.APIKeyProvided, "a later omitted value must reset transient request presence")
|
||||
}
|
||||
|
||||
@@ -4,16 +4,11 @@ package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/netbirdio/netbird/shared/management/proto"
|
||||
)
|
||||
|
||||
// ErrModelDiscoveryUnavailable is returned when no connected proxy in the
|
||||
// requested cluster can execute a model-discovery request.
|
||||
var ErrModelDiscoveryUnavailable = errors.New("model discovery proxy unavailable")
|
||||
|
||||
// Manager defines the interface for proxy operations
|
||||
type Manager interface {
|
||||
Connect(ctx context.Context, proxyID, sessionID, clusterAddress, ipAddress string, accountID *string, capabilities *Capabilities) (*Proxy, error)
|
||||
@@ -43,7 +38,6 @@ type OIDCValidationConfig struct {
|
||||
// Controller is responsible for managing proxy clusters and routing service updates.
|
||||
type Controller interface {
|
||||
SendServiceUpdateToCluster(ctx context.Context, accountID string, update *proto.ProxyMapping, clusterAddr string)
|
||||
DiscoverModels(ctx context.Context, accountID, clusterAddr string, req *proto.ModelDiscoveryRequest) (*proto.ModelDiscoveryResult, error)
|
||||
GetOIDCValidationConfig() OIDCValidationConfig
|
||||
RegisterProxyToCluster(ctx context.Context, clusterAddr, proxyID string) error
|
||||
UnregisterProxyFromCluster(ctx context.Context, clusterAddr, proxyID string) error
|
||||
|
||||
@@ -39,12 +39,6 @@ func (c *GRPCController) SendServiceUpdateToCluster(ctx context.Context, account
|
||||
c.metrics.IncrementServiceUpdateSendCount(clusterAddr)
|
||||
}
|
||||
|
||||
// DiscoverModels executes a correlated model-discovery request on one capable
|
||||
// proxy connected to the requested cluster.
|
||||
func (c *GRPCController) DiscoverModels(ctx context.Context, accountID, clusterAddr string, req *proto.ModelDiscoveryRequest) (*proto.ModelDiscoveryResult, error) {
|
||||
return c.proxyGRPCServer.DiscoverModels(ctx, accountID, clusterAddr, req)
|
||||
}
|
||||
|
||||
// GetOIDCValidationConfig returns the OIDC validation configuration from the gRPC server.
|
||||
func (c *GRPCController) GetOIDCValidationConfig() proxy.OIDCValidationConfig {
|
||||
return c.proxyGRPCServer.GetOIDCValidationConfig()
|
||||
@@ -56,12 +50,10 @@ func (c *GRPCController) RegisterProxyToCluster(ctx context.Context, clusterAddr
|
||||
return nil
|
||||
}
|
||||
proxySet, _ := c.clusterProxies.LoadOrStore(clusterAddr, &sync.Map{})
|
||||
_, alreadyRegistered := proxySet.(*sync.Map).LoadOrStore(proxyID, struct{}{})
|
||||
proxySet.(*sync.Map).Store(proxyID, struct{}{})
|
||||
log.WithContext(ctx).Debugf("Registered proxy %s to cluster %s", proxyID, clusterAddr)
|
||||
|
||||
if !alreadyRegistered {
|
||||
c.metrics.IncrementProxyConnectionCount(clusterAddr)
|
||||
}
|
||||
c.metrics.IncrementProxyConnectionCount(clusterAddr)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -72,10 +64,10 @@ func (c *GRPCController) UnregisterProxyFromCluster(ctx context.Context, cluster
|
||||
return nil
|
||||
}
|
||||
if proxySet, ok := c.clusterProxies.Load(clusterAddr); ok {
|
||||
if _, registered := proxySet.(*sync.Map).LoadAndDelete(proxyID); registered {
|
||||
log.WithContext(ctx).Debugf("Unregistered proxy %s from cluster %s", proxyID, clusterAddr)
|
||||
c.metrics.DecrementProxyConnectionCount(clusterAddr)
|
||||
}
|
||||
proxySet.(*sync.Map).Delete(proxyID)
|
||||
log.WithContext(ctx).Debugf("Unregistered proxy %s from cluster %s", proxyID, clusterAddr)
|
||||
|
||||
c.metrics.DecrementProxyConnectionCount(clusterAddr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -259,21 +259,6 @@ func (m *MockController) EXPECT() *MockControllerMockRecorder {
|
||||
return m.recorder
|
||||
}
|
||||
|
||||
// DiscoverModels mocks base method.
|
||||
func (m *MockController) DiscoverModels(ctx context.Context, accountID, clusterAddr string, req *proto.ModelDiscoveryRequest) (*proto.ModelDiscoveryResult, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "DiscoverModels", ctx, accountID, clusterAddr, req)
|
||||
ret0, _ := ret[0].(*proto.ModelDiscoveryResult)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// DiscoverModels indicates an expected call of DiscoverModels.
|
||||
func (mr *MockControllerMockRecorder) DiscoverModels(ctx, accountID, clusterAddr, req interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiscoverModels", reflect.TypeOf((*MockController)(nil).DiscoverModels), ctx, accountID, clusterAddr, req)
|
||||
}
|
||||
|
||||
// GetOIDCValidationConfig mocks base method.
|
||||
func (m *MockController) GetOIDCValidationConfig() OIDCValidationConfig {
|
||||
m.ctrl.T.Helper()
|
||||
|
||||
@@ -1,404 +0,0 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
rpproxy "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy"
|
||||
"github.com/netbirdio/netbird/shared/management/proto"
|
||||
nbstatus "github.com/netbirdio/netbird/shared/management/status"
|
||||
)
|
||||
|
||||
type modelDiscoveryCall struct {
|
||||
result *proto.ModelDiscoveryResult
|
||||
err error
|
||||
}
|
||||
|
||||
func newModelDiscoveryTestServer(t *testing.T) (*ProxyServiceServer, *testProxyController) {
|
||||
t.Helper()
|
||||
controller := newTestProxyController()
|
||||
server := &ProxyServiceServer{
|
||||
proxyController: controller,
|
||||
modelDiscoveryPending: make(map[string]*pendingModelDiscovery),
|
||||
}
|
||||
return server, controller
|
||||
}
|
||||
|
||||
func addModelDiscoveryTestConnection(
|
||||
t *testing.T,
|
||||
server *ProxyServiceServer,
|
||||
controller *testProxyController,
|
||||
proxyID, sessionID, cluster string,
|
||||
accountID *string,
|
||||
supportsDiscovery bool,
|
||||
) (*proxyConnection, context.CancelFunc) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ready := make(chan struct{})
|
||||
close(ready)
|
||||
conn := &proxyConnection{
|
||||
proxyID: proxyID,
|
||||
sessionID: sessionID,
|
||||
address: cluster,
|
||||
accountID: accountID,
|
||||
capabilities: &proto.ProxyCapabilities{
|
||||
SupportsModelDiscovery: &supportsDiscovery,
|
||||
},
|
||||
syncStream: &syncRecordingStream{},
|
||||
modelDiscoveryChan: make(chan *proto.ModelDiscoveryRequest, modelDiscoveryQueueSize),
|
||||
ready: ready,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
server.connectedProxies.Store(proxyID, conn)
|
||||
require.NoError(t, controller.RegisterProxyToCluster(context.Background(), cluster, proxyID))
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
server.connectedProxies.Delete(proxyID)
|
||||
_ = controller.UnregisterProxyFromCluster(context.Background(), cluster, proxyID)
|
||||
})
|
||||
return conn, cancel
|
||||
}
|
||||
|
||||
func callDiscoverModels(
|
||||
server *ProxyServiceServer,
|
||||
ctx context.Context,
|
||||
accountID, cluster string,
|
||||
req *proto.ModelDiscoveryRequest,
|
||||
) <-chan modelDiscoveryCall {
|
||||
done := make(chan modelDiscoveryCall, 1)
|
||||
go func() {
|
||||
result, err := server.DiscoverModels(ctx, accountID, cluster, req)
|
||||
done <- modelDiscoveryCall{result: result, err: err}
|
||||
}()
|
||||
return done
|
||||
}
|
||||
|
||||
func TestDiscoverModels_CorrelatesResultAndCopiesSensitiveRequest(t *testing.T) {
|
||||
server, controller := newModelDiscoveryTestServer(t)
|
||||
conn, _ := addModelDiscoveryTestConnection(
|
||||
t, server, controller, "proxy-a", "session-a", "cluster.example.com", nil, true,
|
||||
)
|
||||
|
||||
callerReq := &proto.ModelDiscoveryRequest{
|
||||
RequestId: "caller-controlled",
|
||||
UpstreamUrl: "http://ollama.internal:11434",
|
||||
AuthHeaderName: "Authorization",
|
||||
AuthHeaderValue: "Bearer secret",
|
||||
SkipTlsVerify: true,
|
||||
OllamaFallback: true,
|
||||
}
|
||||
done := callDiscoverModels(server, context.Background(), "account-a", "cluster.example.com", callerReq)
|
||||
|
||||
wireReq := <-conn.modelDiscoveryChan
|
||||
assert.NotEmpty(t, wireReq.GetRequestId())
|
||||
assert.NotEqual(t, callerReq.GetRequestId(), wireReq.GetRequestId(), "management must own correlation IDs")
|
||||
assert.Equal(t, callerReq.GetUpstreamUrl(), wireReq.GetUpstreamUrl())
|
||||
assert.Equal(t, callerReq.GetAuthHeaderName(), wireReq.GetAuthHeaderName())
|
||||
assert.Equal(t, callerReq.GetAuthHeaderValue(), wireReq.GetAuthHeaderValue())
|
||||
assert.Equal(t, callerReq.GetSkipTlsVerify(), wireReq.GetSkipTlsVerify())
|
||||
assert.Equal(t, callerReq.GetOllamaFallback(), wireReq.GetOllamaFallback())
|
||||
assert.Equal(t, "caller-controlled", callerReq.GetRequestId(), "caller request must not be mutated")
|
||||
|
||||
want := &proto.ModelDiscoveryResult{
|
||||
RequestId: wireReq.GetRequestId(),
|
||||
Models: []*proto.ModelDiscoveryModel{
|
||||
{Id: "llama3.2:latest", Label: "llama3.2:latest"},
|
||||
},
|
||||
Source: "openai_v1_models",
|
||||
}
|
||||
server.completeModelDiscovery(conn, want)
|
||||
|
||||
got := <-done
|
||||
require.NoError(t, got.err)
|
||||
assert.Equal(t, want, got.result)
|
||||
}
|
||||
|
||||
func TestDiscoverModels_IgnoresMismatchedCorrelation(t *testing.T) {
|
||||
server, controller := newModelDiscoveryTestServer(t)
|
||||
conn, _ := addModelDiscoveryTestConnection(
|
||||
t, server, controller, "proxy-a", "session-a", "cluster.example.com", nil, true,
|
||||
)
|
||||
|
||||
done := callDiscoverModels(server, context.Background(), "account-a", "cluster.example.com", &proto.ModelDiscoveryRequest{
|
||||
UpstreamUrl: "http://ollama.internal:11434",
|
||||
})
|
||||
wireReq := <-conn.modelDiscoveryChan
|
||||
|
||||
server.completeModelDiscovery(conn, &proto.ModelDiscoveryResult{RequestId: "wrong-request"})
|
||||
select {
|
||||
case got := <-done:
|
||||
t.Fatalf("mismatched request completed discovery: %+v", got)
|
||||
default:
|
||||
}
|
||||
|
||||
wrongSession := *conn
|
||||
wrongSession.sessionID = "session-b"
|
||||
server.completeModelDiscovery(&wrongSession, &proto.ModelDiscoveryResult{RequestId: wireReq.GetRequestId()})
|
||||
select {
|
||||
case got := <-done:
|
||||
t.Fatalf("mismatched proxy session completed discovery: %+v", got)
|
||||
default:
|
||||
}
|
||||
|
||||
server.completeModelDiscovery(conn, &proto.ModelDiscoveryResult{RequestId: wireReq.GetRequestId()})
|
||||
got := <-done
|
||||
require.NoError(t, got.err)
|
||||
require.NotNil(t, got.result)
|
||||
}
|
||||
|
||||
func TestDiscoverModels_ReturnsProxyReportedErrorForManagerContext(t *testing.T) {
|
||||
server, controller := newModelDiscoveryTestServer(t)
|
||||
conn, _ := addModelDiscoveryTestConnection(
|
||||
t, server, controller, "proxy-a", "session-a", "cluster.example.com", nil, true,
|
||||
)
|
||||
|
||||
done := callDiscoverModels(server, context.Background(), "account-a", "cluster.example.com", &proto.ModelDiscoveryRequest{
|
||||
UpstreamUrl: "http://ollama.internal:11434",
|
||||
})
|
||||
wireReq := <-conn.modelDiscoveryChan
|
||||
server.completeModelDiscovery(conn, &proto.ModelDiscoveryResult{
|
||||
RequestId: wireReq.GetRequestId(),
|
||||
Error: "upstream returned HTTP 401",
|
||||
})
|
||||
|
||||
got := <-done
|
||||
require.NoError(t, got.err)
|
||||
require.NotNil(t, got.result)
|
||||
assert.Equal(t, "upstream returned HTTP 401", got.result.GetError())
|
||||
assert.Equal(t, wireReq.GetRequestId(), got.result.GetRequestId())
|
||||
}
|
||||
|
||||
func TestDiscoverModels_FiltersCapabilityAndAccountScope(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
connectionAccount *string
|
||||
requestAccount string
|
||||
supported bool
|
||||
}{
|
||||
{
|
||||
name: "capability absent",
|
||||
requestAccount: "account-a",
|
||||
supported: false,
|
||||
},
|
||||
{
|
||||
name: "BYOP account mismatch",
|
||||
connectionAccount: stringPointer("account-b"),
|
||||
requestAccount: "account-a",
|
||||
supported: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
server, controller := newModelDiscoveryTestServer(t)
|
||||
addModelDiscoveryTestConnection(
|
||||
t, server, controller, "proxy-a", "session-a", "cluster.example.com",
|
||||
tt.connectionAccount, tt.supported,
|
||||
)
|
||||
|
||||
result, err := server.DiscoverModels(
|
||||
context.Background(),
|
||||
tt.requestAccount,
|
||||
"cluster.example.com",
|
||||
&proto.ModelDiscoveryRequest{UpstreamUrl: "http://ollama.internal:11434"},
|
||||
)
|
||||
require.Nil(t, result)
|
||||
requireStatusType(t, err, nbstatus.PreconditionFailed)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverModels_RejectsStaleClusterMembershipAfterReconnect(t *testing.T) {
|
||||
server, controller := newModelDiscoveryTestServer(t)
|
||||
conn, _ := addModelDiscoveryTestConnection(
|
||||
t, server, controller, "proxy-a", "new-session", "new-cluster.example.com", nil, true,
|
||||
)
|
||||
// Simulate the stale membership left behind when this proxy ID previously
|
||||
// connected to another cluster and that superseded session skipped cleanup.
|
||||
require.NoError(t, controller.RegisterProxyToCluster(
|
||||
context.Background(),
|
||||
"old-cluster.example.com",
|
||||
"proxy-a",
|
||||
))
|
||||
|
||||
result, err := server.DiscoverModels(
|
||||
context.Background(),
|
||||
"account-a",
|
||||
"old-cluster.example.com",
|
||||
&proto.ModelDiscoveryRequest{UpstreamUrl: "http://ollama.internal:11434"},
|
||||
)
|
||||
|
||||
require.Nil(t, result)
|
||||
requireStatusType(t, err, nbstatus.PreconditionFailed)
|
||||
select {
|
||||
case request := <-conn.modelDiscoveryChan:
|
||||
t.Fatalf("sent credentials to connection in a different cluster: %+v", request)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverModels_CancelAndDisconnectCleanPendingRequest(t *testing.T) {
|
||||
t.Run("caller cancellation", func(t *testing.T) {
|
||||
server, controller := newModelDiscoveryTestServer(t)
|
||||
conn, _ := addModelDiscoveryTestConnection(
|
||||
t, server, controller, "proxy-a", "session-a", "cluster.example.com", nil, true,
|
||||
)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := callDiscoverModels(server, ctx, "account-a", "cluster.example.com", &proto.ModelDiscoveryRequest{
|
||||
UpstreamUrl: "http://ollama.internal:11434",
|
||||
})
|
||||
<-conn.modelDiscoveryChan
|
||||
cancel()
|
||||
|
||||
got := <-done
|
||||
require.Nil(t, got.result)
|
||||
requireStatusType(t, got.err, nbstatus.PreconditionFailed)
|
||||
assert.Empty(t, server.modelDiscoveryPending)
|
||||
})
|
||||
|
||||
t.Run("proxy disconnect", func(t *testing.T) {
|
||||
server, controller := newModelDiscoveryTestServer(t)
|
||||
conn, _ := addModelDiscoveryTestConnection(
|
||||
t, server, controller, "proxy-a", "session-a", "cluster.example.com", nil, true,
|
||||
)
|
||||
done := callDiscoverModels(server, context.Background(), "account-a", "cluster.example.com", &proto.ModelDiscoveryRequest{
|
||||
UpstreamUrl: "http://ollama.internal:11434",
|
||||
})
|
||||
<-conn.modelDiscoveryChan
|
||||
server.failModelDiscoveries(conn, rpproxy.ErrModelDiscoveryUnavailable)
|
||||
|
||||
got := <-done
|
||||
require.Nil(t, got.result)
|
||||
requireStatusType(t, got.err, nbstatus.PreconditionFailed)
|
||||
assert.Empty(t, server.modelDiscoveryPending)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDiscoverModels_UsesNextCapableProxyWhenFirstQueueIsFull(t *testing.T) {
|
||||
server, controller := newModelDiscoveryTestServer(t)
|
||||
first, _ := addModelDiscoveryTestConnection(
|
||||
t, server, controller, "proxy-a", "session-a", "cluster.example.com", nil, true,
|
||||
)
|
||||
second, _ := addModelDiscoveryTestConnection(
|
||||
t, server, controller, "proxy-b", "session-b", "cluster.example.com", nil, true,
|
||||
)
|
||||
first.modelDiscoveryChan = make(chan *proto.ModelDiscoveryRequest, 1)
|
||||
first.modelDiscoveryChan <- &proto.ModelDiscoveryRequest{RequestId: "occupied"}
|
||||
|
||||
done := callDiscoverModels(server, context.Background(), "account-a", "cluster.example.com", &proto.ModelDiscoveryRequest{
|
||||
UpstreamUrl: "http://ollama.internal:11434",
|
||||
})
|
||||
wireReq := <-second.modelDiscoveryChan
|
||||
server.completeModelDiscovery(second, &proto.ModelDiscoveryResult{RequestId: wireReq.GetRequestId()})
|
||||
|
||||
got := <-done
|
||||
require.NoError(t, got.err)
|
||||
require.NotNil(t, got.result)
|
||||
}
|
||||
|
||||
func TestSenderSkipsCanceledQueuedModelDiscovery(t *testing.T) {
|
||||
server, controller := newModelDiscoveryTestServer(t)
|
||||
conn, _ := addModelDiscoveryTestConnection(
|
||||
t, server, controller, "proxy-a", "session-a", "cluster.example.com", nil, true,
|
||||
)
|
||||
stream := conn.syncStream.(*syncRecordingStream)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := callDiscoverModels(server, ctx, "account-a", "cluster.example.com", &proto.ModelDiscoveryRequest{
|
||||
UpstreamUrl: "http://ollama.internal:11434",
|
||||
})
|
||||
require.Eventually(t, func() bool {
|
||||
return len(conn.modelDiscoveryChan) == 1
|
||||
}, time.Second, time.Millisecond)
|
||||
cancel()
|
||||
got := <-done
|
||||
require.Nil(t, got.result)
|
||||
requireStatusType(t, got.err, nbstatus.PreconditionFailed)
|
||||
|
||||
liveRequest := &proto.ModelDiscoveryRequest{RequestId: "live-request"}
|
||||
livePending := &pendingModelDiscovery{
|
||||
proxyID: conn.proxyID,
|
||||
sessionID: conn.sessionID,
|
||||
done: make(chan modelDiscoveryCompletion, 1),
|
||||
}
|
||||
server.registerModelDiscovery(liveRequest.GetRequestId(), livePending)
|
||||
defer server.removeModelDiscovery(liveRequest.GetRequestId(), livePending)
|
||||
conn.modelDiscoveryChan <- liveRequest
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go server.sender(conn, errCh)
|
||||
require.Eventually(t, func() bool {
|
||||
stream.mu.Lock()
|
||||
defer stream.mu.Unlock()
|
||||
return len(stream.sent) == 1
|
||||
}, time.Second, time.Millisecond)
|
||||
|
||||
stream.mu.Lock()
|
||||
sent := append([]*proto.SyncMappingsResponse(nil), stream.sent...)
|
||||
stream.mu.Unlock()
|
||||
require.Len(t, sent, 1)
|
||||
assert.Equal(t, "live-request", sent[0].GetModelDiscoveryRequest().GetRequestId())
|
||||
}
|
||||
|
||||
func TestDrainRecv_DispatchesModelDiscoveryResult(t *testing.T) {
|
||||
server, _ := newModelDiscoveryTestServer(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
conn := &proxyConnection{
|
||||
proxyID: "proxy-a",
|
||||
sessionID: "session-a",
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
pending := &pendingModelDiscovery{
|
||||
proxyID: conn.proxyID,
|
||||
sessionID: conn.sessionID,
|
||||
done: make(chan modelDiscoveryCompletion, 1),
|
||||
}
|
||||
server.registerModelDiscovery("request-a", pending)
|
||||
stream := &syncRecordingStream{
|
||||
recvMsgs: []*proto.SyncMappingsRequest{
|
||||
{
|
||||
Msg: &proto.SyncMappingsRequest_ModelDiscoveryResult{
|
||||
ModelDiscoveryResult: &proto.ModelDiscoveryResult{RequestId: "request-a"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
go server.drainRecv(conn, stream, errCh)
|
||||
|
||||
completion := <-pending.done
|
||||
require.NoError(t, completion.err)
|
||||
require.NotNil(t, completion.result)
|
||||
assert.Equal(t, "request-a", completion.result.GetRequestId())
|
||||
}
|
||||
|
||||
func TestSendModelDiscoveryRequest_UsesOutOfBandSyncField(t *testing.T) {
|
||||
stream := &syncRecordingStream{}
|
||||
conn := &proxyConnection{syncStream: stream}
|
||||
req := &proto.ModelDiscoveryRequest{RequestId: "request-a"}
|
||||
|
||||
require.NoError(t, conn.sendModelDiscoveryRequest(req))
|
||||
require.Len(t, stream.sent, 1)
|
||||
assert.Empty(t, stream.sent[0].GetMapping())
|
||||
assert.Equal(t, req, stream.sent[0].GetModelDiscoveryRequest())
|
||||
}
|
||||
|
||||
func stringPointer(value string) *string {
|
||||
return &value
|
||||
}
|
||||
|
||||
func requireStatusType(t *testing.T, err error, want nbstatus.Type) {
|
||||
t.Helper()
|
||||
require.Error(t, err)
|
||||
statusErr, ok := nbstatus.FromError(err)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, want, statusErr.Type())
|
||||
}
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -30,7 +29,6 @@ import (
|
||||
|
||||
"github.com/netbirdio/netbird/shared/management/domain"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/peers"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy"
|
||||
@@ -38,6 +36,7 @@ import (
|
||||
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/sessionkey"
|
||||
"github.com/netbirdio/netbird/management/server/idp"
|
||||
"github.com/netbirdio/netbird/management/server/peer"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
|
||||
"github.com/netbirdio/netbird/management/server/types"
|
||||
"github.com/netbirdio/netbird/management/server/users"
|
||||
proxyauth "github.com/netbirdio/netbird/proxy/auth"
|
||||
@@ -85,10 +84,6 @@ type ProxyServiceServer struct {
|
||||
|
||||
// Map of connected proxies: proxy_id -> proxy connection
|
||||
connectedProxies sync.Map
|
||||
// modelDiscoveryPending correlates out-of-band model-discovery results
|
||||
// received on SyncMappings with the management request waiting for them.
|
||||
modelDiscoveryMu sync.Mutex
|
||||
modelDiscoveryPending map[string]*pendingModelDiscovery
|
||||
|
||||
// Manager for access logs
|
||||
accessLogManager accesslogs.Manager
|
||||
@@ -148,8 +143,6 @@ const defaultProxyTokenTTL = 5 * time.Minute
|
||||
|
||||
const defaultSnapshotBatchSize = 500
|
||||
|
||||
const modelDiscoveryQueueSize = 16
|
||||
|
||||
func snapshotBatchSizeFromEnv() int {
|
||||
if v := os.Getenv("NB_PROXY_SNAPSHOT_BATCH_SIZE"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
@@ -178,26 +171,10 @@ type proxyConnection struct {
|
||||
stream proto.ProxyService_GetMappingUpdateServer
|
||||
// syncStream is set when the proxy connected via SyncMappings.
|
||||
// When non-nil, the sender goroutine uses this instead of stream.
|
||||
syncStream proto.ProxyService_SyncMappingsServer
|
||||
sendChan chan *proto.GetMappingUpdateResponse
|
||||
modelDiscoveryChan chan *proto.ModelDiscoveryRequest
|
||||
// ready closes after the initial snapshot completes and the sender
|
||||
// goroutine is about to start. Discovery never competes with snapshot
|
||||
// delivery on the stream.
|
||||
ready chan struct{}
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
type modelDiscoveryCompletion struct {
|
||||
result *proto.ModelDiscoveryResult
|
||||
err error
|
||||
}
|
||||
|
||||
type pendingModelDiscovery struct {
|
||||
proxyID string
|
||||
sessionID string
|
||||
done chan modelDiscoveryCompletion
|
||||
syncStream proto.ProxyService_SyncMappingsServer
|
||||
sendChan chan *proto.GetMappingUpdateResponse
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func enforceAccountScope(ctx context.Context, requestAccountID string) error {
|
||||
@@ -215,18 +192,17 @@ func enforceAccountScope(ctx context.Context, requestAccountID string) error {
|
||||
func NewProxyServiceServer(accessLogMgr accesslogs.Manager, tokenStore *OneTimeTokenStore, pkceStore *PKCEVerifierStore, oidcConfig ProxyOIDCConfig, peersManager peers.Manager, usersManager users.Manager, idpManager idp.Manager, proxyMgr proxy.Manager, tokenChecker ProxyTokenChecker) *ProxyServiceServer {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s := &ProxyServiceServer{
|
||||
accessLogManager: accessLogMgr,
|
||||
oidcConfig: oidcConfig,
|
||||
tokenStore: tokenStore,
|
||||
pkceVerifierStore: pkceStore,
|
||||
peersManager: peersManager,
|
||||
usersManager: usersManager,
|
||||
idpManager: idpManager,
|
||||
proxyManager: proxyMgr,
|
||||
tokenChecker: tokenChecker,
|
||||
snapshotBatchSize: snapshotBatchSizeFromEnv(),
|
||||
modelDiscoveryPending: make(map[string]*pendingModelDiscovery),
|
||||
cancel: cancel,
|
||||
accessLogManager: accessLogMgr,
|
||||
oidcConfig: oidcConfig,
|
||||
tokenStore: tokenStore,
|
||||
pkceVerifierStore: pkceStore,
|
||||
peersManager: peersManager,
|
||||
usersManager: usersManager,
|
||||
idpManager: idpManager,
|
||||
proxyManager: proxyMgr,
|
||||
tokenChecker: tokenChecker,
|
||||
snapshotBatchSize: snapshotBatchSizeFromEnv(),
|
||||
cancel: cancel,
|
||||
}
|
||||
go s.cleanupStaleProxies(ctx)
|
||||
return s
|
||||
@@ -309,6 +285,7 @@ func (s *ProxyServiceServer) CheckLLMPolicyLimits(ctx context.Context, req *prot
|
||||
UserID: req.GetUserId(),
|
||||
GroupIDs: req.GetGroupIds(),
|
||||
ProviderID: req.GetProviderId(),
|
||||
Model: req.GetModel(),
|
||||
})
|
||||
if err != nil {
|
||||
log.WithContext(ctx).Errorf("select policy for request: %v", err)
|
||||
@@ -416,7 +393,6 @@ func (s *ProxyServiceServer) GetMappingUpdate(req *proto.GetMappingUpdateRequest
|
||||
return fmt.Errorf("send snapshot to proxy %s: %w", params.proxyID, err)
|
||||
}
|
||||
|
||||
close(conn.ready)
|
||||
errChan := make(chan error, 2)
|
||||
go s.sender(conn, errChan)
|
||||
|
||||
@@ -450,10 +426,9 @@ func (s *ProxyServiceServer) SyncMappings(stream proto.ProxyService_SyncMappings
|
||||
return fmt.Errorf("send snapshot to proxy %s: %w", params.proxyID, err)
|
||||
}
|
||||
|
||||
close(conn.ready)
|
||||
errChan := make(chan error, 2)
|
||||
go s.sender(conn, errChan)
|
||||
go s.drainRecv(conn, stream, errChan)
|
||||
go s.drainRecv(stream, errChan)
|
||||
|
||||
return s.serveProxyConnection(conn, proxyRecord, errChan, true)
|
||||
}
|
||||
@@ -522,8 +497,6 @@ func (s *ProxyServiceServer) registerProxyConnection(ctx context.Context, params
|
||||
connSeed.tokenID = tokenID
|
||||
connSeed.capabilities = params.capabilities
|
||||
connSeed.sendChan = make(chan *proto.GetMappingUpdateResponse, 100)
|
||||
connSeed.modelDiscoveryChan = make(chan *proto.ModelDiscoveryRequest, modelDiscoveryQueueSize)
|
||||
connSeed.ready = make(chan struct{})
|
||||
connSeed.ctx = connCtx
|
||||
connSeed.cancel = cancel
|
||||
|
||||
@@ -571,13 +544,10 @@ func (s *ProxyServiceServer) supersedePriorConnection(proxyID, newSessionID stri
|
||||
// cleanupFailedSnapshot removes the connection from the cluster and store
|
||||
// after a snapshot send failure.
|
||||
func (s *ProxyServiceServer) cleanupFailedSnapshot(ctx context.Context, conn *proxyConnection) {
|
||||
s.failModelDiscoveries(conn, proxy.ErrModelDiscoveryUnavailable)
|
||||
if s.connectedProxies.CompareAndDelete(conn.proxyID, conn) {
|
||||
if err := s.proxyController.UnregisterProxyFromCluster(context.Background(), conn.address, conn.proxyID); err != nil {
|
||||
log.WithContext(ctx).Debugf("cleanup after snapshot failure for proxy %s: %v", conn.proxyID, err)
|
||||
}
|
||||
} else {
|
||||
s.unregisterSupersededProxyCluster(ctx, conn)
|
||||
}
|
||||
conn.cancel()
|
||||
if err := s.proxyManager.Disconnect(context.Background(), conn.proxyID, conn.sessionID); err != nil {
|
||||
@@ -585,19 +555,15 @@ func (s *ProxyServiceServer) cleanupFailedSnapshot(ctx context.Context, conn *pr
|
||||
}
|
||||
}
|
||||
|
||||
// drainRecv consumes post-snapshot messages from a bidirectional stream.
|
||||
// Incremental mapping acks need no action; model-discovery results are
|
||||
// dispatched to the correlated management caller.
|
||||
func (s *ProxyServiceServer) drainRecv(conn *proxyConnection, stream proto.ProxyService_SyncMappingsServer, errChan chan<- error) {
|
||||
// drainRecv consumes and discards messages from a bidirectional stream.
|
||||
// The proxy sends an ack for every incremental update; we don't need them
|
||||
// after the snapshot phase. Recv errors are forwarded to errChan.
|
||||
func (s *ProxyServiceServer) drainRecv(stream proto.ProxyService_SyncMappingsServer, errChan chan<- error) {
|
||||
for {
|
||||
msg, err := stream.Recv()
|
||||
if err != nil {
|
||||
if _, err := stream.Recv(); err != nil {
|
||||
errChan <- err
|
||||
return
|
||||
}
|
||||
if result := msg.GetModelDiscoveryResult(); result != nil {
|
||||
s.completeModelDiscovery(conn, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -634,9 +600,7 @@ func (s *ProxyServiceServer) serveProxyConnection(conn *proxyConnection, proxyRe
|
||||
// disconnectProxy removes the connection from cluster and store, unless it
|
||||
// has already been superseded by a newer connection.
|
||||
func (s *ProxyServiceServer) disconnectProxy(conn *proxyConnection) {
|
||||
s.failModelDiscoveries(conn, proxy.ErrModelDiscoveryUnavailable)
|
||||
if !s.connectedProxies.CompareAndDelete(conn.proxyID, conn) {
|
||||
s.unregisterSupersededProxyCluster(context.Background(), conn)
|
||||
log.Infof("Proxy %s session %s: skipping cleanup, superseded by new connection", conn.proxyID, conn.sessionID)
|
||||
conn.cancel()
|
||||
return
|
||||
@@ -653,26 +617,6 @@ func (s *ProxyServiceServer) disconnectProxy(conn *proxyConnection) {
|
||||
log.Infof("Proxy %s session %s disconnected", conn.proxyID, conn.sessionID)
|
||||
}
|
||||
|
||||
// unregisterSupersededProxyCluster removes membership owned only by an old
|
||||
// session after the same proxy ID reconnects at a different address. When the
|
||||
// address is unchanged, the membership is shared with the replacement and
|
||||
// must remain registered.
|
||||
func (s *ProxyServiceServer) unregisterSupersededProxyCluster(ctx context.Context, conn *proxyConnection) {
|
||||
if conn == nil || s.proxyController == nil {
|
||||
return
|
||||
}
|
||||
currentValue, ok := s.connectedProxies.Load(conn.proxyID)
|
||||
if ok {
|
||||
current := currentValue.(*proxyConnection)
|
||||
if current == conn || current.address == conn.address {
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := s.proxyController.UnregisterProxyFromCluster(ctx, conn.address, conn.proxyID); err != nil {
|
||||
log.WithContext(ctx).Debugf("cleanup superseded cluster membership for proxy %s: %v", conn.proxyID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// sendSnapshotSync sends the initial snapshot with back-pressure: it sends
|
||||
// one batch, then waits for the proxy to ack before sending the next.
|
||||
func (s *ProxyServiceServer) sendSnapshotSync(ctx context.Context, conn *proxyConnection, stream proto.ProxyService_SyncMappingsServer) error {
|
||||
@@ -909,20 +853,6 @@ func (s *ProxyServiceServer) sender(conn *proxyConnection, errChan chan<- error)
|
||||
return
|
||||
}
|
||||
log.WithContext(conn.ctx).Tracef("Send response to proxy %s", conn.proxyID)
|
||||
case req := <-conn.modelDiscoveryChan:
|
||||
// The API caller may have timed out while this request waited
|
||||
// behind other stream work. Pending correlation is removed on
|
||||
// cancellation, so skip stale probes before they reach the
|
||||
// proxy and expose a stored credential unnecessarily.
|
||||
if !s.isModelDiscoveryPendingFor(conn, req.GetRequestId()) {
|
||||
continue
|
||||
}
|
||||
if err := conn.sendModelDiscoveryRequest(req); err != nil {
|
||||
errChan <- err
|
||||
log.WithContext(conn.ctx).Tracef("Failed to send model discovery request to proxy %s: %v", conn.proxyID, err)
|
||||
return
|
||||
}
|
||||
log.WithContext(conn.ctx).Tracef("Sent model discovery request to proxy %s", conn.proxyID)
|
||||
case <-conn.ctx.Done():
|
||||
return
|
||||
}
|
||||
@@ -940,15 +870,6 @@ func (conn *proxyConnection) sendResponse(resp *proto.GetMappingUpdateResponse)
|
||||
return conn.stream.Send(resp)
|
||||
}
|
||||
|
||||
func (conn *proxyConnection) sendModelDiscoveryRequest(req *proto.ModelDiscoveryRequest) error {
|
||||
if conn.syncStream == nil {
|
||||
return proxy.ErrModelDiscoveryUnavailable
|
||||
}
|
||||
return conn.syncStream.Send(&proto.SyncMappingsResponse{
|
||||
ModelDiscoveryRequest: req,
|
||||
})
|
||||
}
|
||||
|
||||
// SendAccessLog processes access log from proxy
|
||||
func (s *ProxyServiceServer) SendAccessLog(ctx context.Context, req *proto.SendAccessLogRequest) (*proto.SendAccessLogResponse, error) {
|
||||
accessLog := req.GetLog()
|
||||
@@ -1097,181 +1018,6 @@ func (s *ProxyServiceServer) GetConnectedProxyURLs() []string {
|
||||
return urls
|
||||
}
|
||||
|
||||
// DiscoverModels sends one correlated model-discovery request to a capable
|
||||
// proxy in clusterAddr and waits for its result. Only the stored connection's
|
||||
// cluster/account scope is used for routing; no routing identity is carried in
|
||||
// the request delivered to the proxy.
|
||||
func (s *ProxyServiceServer) DiscoverModels(ctx context.Context, accountID, clusterAddr string, req *proto.ModelDiscoveryRequest) (*proto.ModelDiscoveryResult, error) {
|
||||
if req == nil {
|
||||
return nil, nbstatus.Errorf(nbstatus.InvalidArgument, "model discovery request is required")
|
||||
}
|
||||
if strings.TrimSpace(accountID) == "" {
|
||||
return nil, nbstatus.Errorf(nbstatus.InvalidArgument, "account ID is required for model discovery")
|
||||
}
|
||||
if strings.TrimSpace(clusterAddr) == "" {
|
||||
return nil, nbstatus.Errorf(nbstatus.PreconditionFailed, "agent network proxy cluster is not configured")
|
||||
}
|
||||
if s.proxyController == nil {
|
||||
return nil, nbstatus.Errorf(nbstatus.PreconditionFailed, "model discovery is unavailable for the configured proxy cluster")
|
||||
}
|
||||
|
||||
proxyIDs := s.proxyController.GetProxiesForCluster(clusterAddr)
|
||||
sort.Strings(proxyIDs)
|
||||
|
||||
request := &proto.ModelDiscoveryRequest{
|
||||
RequestId: uuid.NewString(),
|
||||
UpstreamUrl: req.GetUpstreamUrl(),
|
||||
AuthHeaderName: req.GetAuthHeaderName(),
|
||||
AuthHeaderValue: req.GetAuthHeaderValue(),
|
||||
SkipTlsVerify: req.GetSkipTlsVerify(),
|
||||
OllamaFallback: req.GetOllamaFallback(),
|
||||
}
|
||||
|
||||
for _, proxyID := range proxyIDs {
|
||||
connVal, ok := s.connectedProxies.Load(proxyID)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
conn := connVal.(*proxyConnection)
|
||||
// Cluster membership can briefly retain a proxy ID from a superseded
|
||||
// session. The live connection is authoritative: never send provider
|
||||
// credentials unless it is connected to the exact requested cluster.
|
||||
if conn.address != clusterAddr {
|
||||
continue
|
||||
}
|
||||
if !modelDiscoveryCapable(conn, accountID) {
|
||||
continue
|
||||
}
|
||||
|
||||
pending := &pendingModelDiscovery{
|
||||
proxyID: conn.proxyID,
|
||||
sessionID: conn.sessionID,
|
||||
done: make(chan modelDiscoveryCompletion, 1),
|
||||
}
|
||||
s.registerModelDiscovery(request.GetRequestId(), pending)
|
||||
|
||||
queued := false
|
||||
select {
|
||||
case conn.modelDiscoveryChan <- request:
|
||||
queued = true
|
||||
case <-ctx.Done():
|
||||
s.removeModelDiscovery(request.GetRequestId(), pending)
|
||||
return nil, modelDiscoveryContextError(ctx.Err())
|
||||
default:
|
||||
s.removeModelDiscovery(request.GetRequestId(), pending)
|
||||
}
|
||||
if !queued {
|
||||
continue
|
||||
}
|
||||
|
||||
defer s.removeModelDiscovery(request.GetRequestId(), pending)
|
||||
select {
|
||||
case completion := <-pending.done:
|
||||
if completion.err != nil {
|
||||
return nil, nbstatus.Errorf(nbstatus.PreconditionFailed, "model discovery proxy disconnected")
|
||||
}
|
||||
return completion.result, nil
|
||||
case <-ctx.Done():
|
||||
return nil, modelDiscoveryContextError(ctx.Err())
|
||||
case <-conn.ctx.Done():
|
||||
return nil, nbstatus.Errorf(nbstatus.PreconditionFailed, "model discovery proxy disconnected")
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nbstatus.Errorf(nbstatus.PreconditionFailed, "no connected proxy in the configured cluster supports model discovery")
|
||||
}
|
||||
|
||||
func modelDiscoveryCapable(conn *proxyConnection, accountID string) bool {
|
||||
if conn == nil || conn.syncStream == nil || conn.modelDiscoveryChan == nil || conn.ready == nil {
|
||||
return false
|
||||
}
|
||||
if conn.ctx == nil || conn.ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
if conn.accountID != nil && *conn.accountID != accountID {
|
||||
return false
|
||||
}
|
||||
if conn.capabilities == nil || !conn.capabilities.GetSupportsModelDiscovery() {
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case <-conn.ready:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func modelDiscoveryContextError(err error) error {
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return nbstatus.Errorf(nbstatus.PreconditionFailed, "model discovery timed out")
|
||||
}
|
||||
return nbstatus.Errorf(nbstatus.PreconditionFailed, "model discovery was canceled")
|
||||
}
|
||||
|
||||
func (s *ProxyServiceServer) registerModelDiscovery(requestID string, pending *pendingModelDiscovery) {
|
||||
s.modelDiscoveryMu.Lock()
|
||||
defer s.modelDiscoveryMu.Unlock()
|
||||
if s.modelDiscoveryPending == nil {
|
||||
s.modelDiscoveryPending = make(map[string]*pendingModelDiscovery)
|
||||
}
|
||||
s.modelDiscoveryPending[requestID] = pending
|
||||
}
|
||||
|
||||
func (s *ProxyServiceServer) removeModelDiscovery(requestID string, pending *pendingModelDiscovery) {
|
||||
s.modelDiscoveryMu.Lock()
|
||||
defer s.modelDiscoveryMu.Unlock()
|
||||
if s.modelDiscoveryPending[requestID] == pending {
|
||||
delete(s.modelDiscoveryPending, requestID)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ProxyServiceServer) isModelDiscoveryPendingFor(conn *proxyConnection, requestID string) bool {
|
||||
if conn == nil || requestID == "" {
|
||||
return false
|
||||
}
|
||||
s.modelDiscoveryMu.Lock()
|
||||
defer s.modelDiscoveryMu.Unlock()
|
||||
pending := s.modelDiscoveryPending[requestID]
|
||||
return pending != nil && pending.proxyID == conn.proxyID && pending.sessionID == conn.sessionID
|
||||
}
|
||||
|
||||
func (s *ProxyServiceServer) completeModelDiscovery(conn *proxyConnection, result *proto.ModelDiscoveryResult) {
|
||||
if conn == nil || result == nil || result.GetRequestId() == "" {
|
||||
return
|
||||
}
|
||||
s.modelDiscoveryMu.Lock()
|
||||
pending := s.modelDiscoveryPending[result.GetRequestId()]
|
||||
s.modelDiscoveryMu.Unlock()
|
||||
if pending == nil || pending.proxyID != conn.proxyID || pending.sessionID != conn.sessionID {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case pending.done <- modelDiscoveryCompletion{result: result}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ProxyServiceServer) failModelDiscoveries(conn *proxyConnection, err error) {
|
||||
if conn == nil {
|
||||
return
|
||||
}
|
||||
s.modelDiscoveryMu.Lock()
|
||||
pending := make([]*pendingModelDiscovery, 0)
|
||||
for _, item := range s.modelDiscoveryPending {
|
||||
if item.proxyID == conn.proxyID && item.sessionID == conn.sessionID {
|
||||
pending = append(pending, item)
|
||||
}
|
||||
}
|
||||
s.modelDiscoveryMu.Unlock()
|
||||
for _, item := range pending {
|
||||
select {
|
||||
case item.done <- modelDiscoveryCompletion{err: err}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SendServiceUpdateToCluster sends a service update to all proxy servers in a specific cluster.
|
||||
// If clusterAddr is empty, broadcasts to all connected proxy servers (backward compatibility).
|
||||
// For create/update operations a unique one-time auth token is generated per
|
||||
@@ -1304,12 +1050,6 @@ func (s *ProxyServiceServer) SendServiceUpdateToCluster(ctx context.Context, upd
|
||||
continue
|
||||
}
|
||||
conn := connVal.(*proxyConnection)
|
||||
// Membership can retain this proxy ID from a superseded session in a
|
||||
// different cluster. The live connection address is authoritative,
|
||||
// especially because Agent Network mappings can contain credentials.
|
||||
if conn.address != clusterAddr {
|
||||
continue
|
||||
}
|
||||
if conn.accountID != nil && update.AccountId != "" && *conn.accountID != update.AccountId {
|
||||
continue
|
||||
}
|
||||
|
||||
138
management/internals/shared/grpc/proxy_llm_policy_limits_test.go
Normal file
138
management/internals/shared/grpc/proxy_llm_policy_limits_test.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package grpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc/codes"
|
||||
grpcstatus "google.golang.org/grpc/status"
|
||||
|
||||
"github.com/netbirdio/netbird/management/internals/modules/agentnetwork"
|
||||
"github.com/netbirdio/netbird/shared/management/proto"
|
||||
)
|
||||
|
||||
// fakeAgentNetworkLimits records the PolicySelectionInput it was invoked with
|
||||
// and returns a pre-programmed result, so tests can assert what the handler
|
||||
// forwards to the selector.
|
||||
type fakeAgentNetworkLimits struct {
|
||||
gotInput agentnetwork.PolicySelectionInput
|
||||
result *agentnetwork.PolicySelectionResult
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeAgentNetworkLimits) SelectPolicyForRequest(_ context.Context, in agentnetwork.PolicySelectionInput) (*agentnetwork.PolicySelectionResult, error) {
|
||||
f.gotInput = in
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return f.result, nil
|
||||
}
|
||||
|
||||
func (f *fakeAgentNetworkLimits) RecordUsage(_ context.Context, _ agentnetwork.RecordUsageInput) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestCheckLLMPolicyLimits_ForwardsModelToSelector proves the wiring added here:
|
||||
// the model the proxy extracted must reach the selector's Model unchanged,
|
||||
// alongside the account/user/group/provider fields.
|
||||
func TestCheckLLMPolicyLimits_ForwardsModelToSelector(t *testing.T) {
|
||||
fake := &fakeAgentNetworkLimits{result: &agentnetwork.PolicySelectionResult{Allow: true, SelectedPolicyID: "pol-1"}}
|
||||
s := &ProxyServiceServer{}
|
||||
s.SetAgentNetworkLimitsService(fake)
|
||||
|
||||
req := &proto.CheckLLMPolicyLimitsRequest{
|
||||
AccountId: "acc-1",
|
||||
UserId: "user-1",
|
||||
GroupIds: []string{"grp-a", "grp-b"},
|
||||
ProviderId: "prov-1",
|
||||
Model: "claude-opus-4",
|
||||
}
|
||||
|
||||
resp, err := s.CheckLLMPolicyLimits(context.Background(), req)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resp)
|
||||
|
||||
assert.Equal(t, "acc-1", fake.gotInput.AccountID)
|
||||
assert.Equal(t, "user-1", fake.gotInput.UserID)
|
||||
assert.Equal(t, []string{"grp-a", "grp-b"}, fake.gotInput.GroupIDs)
|
||||
assert.Equal(t, "prov-1", fake.gotInput.ProviderID)
|
||||
assert.Equal(t, "claude-opus-4", fake.gotInput.Model,
|
||||
"the request's model must be forwarded to the selector")
|
||||
}
|
||||
|
||||
// TestCheckLLMPolicyLimits_EmptyModelForwardedAsEmpty proves an undetermined
|
||||
// model (empty string) is forwarded as-is; the selector decides how to treat it.
|
||||
func TestCheckLLMPolicyLimits_EmptyModelForwardedAsEmpty(t *testing.T) {
|
||||
fake := &fakeAgentNetworkLimits{result: &agentnetwork.PolicySelectionResult{Allow: true}}
|
||||
s := &ProxyServiceServer{}
|
||||
s.SetAgentNetworkLimitsService(fake)
|
||||
|
||||
req := &proto.CheckLLMPolicyLimitsRequest{
|
||||
AccountId: "acc-1",
|
||||
ProviderId: "prov-1",
|
||||
}
|
||||
|
||||
_, err := s.CheckLLMPolicyLimits(context.Background(), req)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", fake.gotInput.Model, "an absent model must be forwarded as empty, not fabricated")
|
||||
}
|
||||
|
||||
// TestCheckLLMPolicyLimits_DenyResponseCarriesModelBlockedCode proves the deny
|
||||
// envelope surfaces the model-allowlist deny code + reason through the response.
|
||||
func TestCheckLLMPolicyLimits_DenyResponseCarriesModelBlockedCode(t *testing.T) {
|
||||
fake := &fakeAgentNetworkLimits{result: &agentnetwork.PolicySelectionResult{
|
||||
Allow: false,
|
||||
DenyCode: "llm_policy.model_blocked",
|
||||
DenyReason: `model "claude-opus-4" is not permitted by any applicable policy allowlist`,
|
||||
}}
|
||||
s := &ProxyServiceServer{}
|
||||
s.SetAgentNetworkLimitsService(fake)
|
||||
|
||||
resp, err := s.CheckLLMPolicyLimits(context.Background(), &proto.CheckLLMPolicyLimitsRequest{
|
||||
AccountId: "acc-1",
|
||||
ProviderId: "prov-1",
|
||||
Model: "claude-opus-4",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resp)
|
||||
assert.Equal(t, "deny", resp.Decision)
|
||||
assert.Equal(t, "llm_policy.model_blocked", resp.DenyCode)
|
||||
assert.NotEmpty(t, resp.DenyReason)
|
||||
assert.Empty(t, resp.SelectedPolicyId, "a denied request must carry no selected policy")
|
||||
}
|
||||
|
||||
// TestCheckLLMPolicyLimits_SelectorErrorSurfacesAsInternal proves a selector
|
||||
// failure surfaces as an Internal gRPC error rather than a silent allow.
|
||||
func TestCheckLLMPolicyLimits_SelectorErrorSurfacesAsInternal(t *testing.T) {
|
||||
fake := &fakeAgentNetworkLimits{err: errors.New("boom")}
|
||||
s := &ProxyServiceServer{}
|
||||
s.SetAgentNetworkLimitsService(fake)
|
||||
|
||||
_, err := s.CheckLLMPolicyLimits(context.Background(), &proto.CheckLLMPolicyLimitsRequest{
|
||||
AccountId: "acc-1",
|
||||
ProviderId: "prov-1",
|
||||
Model: "gpt-4o",
|
||||
})
|
||||
require.Error(t, err)
|
||||
st, ok := grpcstatus.FromError(err)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, codes.Internal, st.Code(), "selector errors must never fail open on the hot path")
|
||||
}
|
||||
|
||||
// TestCheckLLMPolicyLimits_UnconfiguredServiceReturnsUnimplemented locks the
|
||||
// fallback: with no limits service wired the RPC returns Unimplemented.
|
||||
func TestCheckLLMPolicyLimits_UnconfiguredServiceReturnsUnimplemented(t *testing.T) {
|
||||
s := &ProxyServiceServer{}
|
||||
|
||||
_, err := s.CheckLLMPolicyLimits(context.Background(), &proto.CheckLLMPolicyLimitsRequest{
|
||||
AccountId: "acc-1",
|
||||
ProviderId: "prov-1",
|
||||
})
|
||||
require.Error(t, err)
|
||||
st, ok := grpcstatus.FromError(err)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, codes.Unimplemented, st.Code())
|
||||
}
|
||||
@@ -42,10 +42,6 @@ func newTestProxyController() *testProxyController {
|
||||
func (c *testProxyController) SendServiceUpdateToCluster(_ context.Context, _ string, _ *proto.ProxyMapping, _ string) {
|
||||
}
|
||||
|
||||
func (c *testProxyController) DiscoverModels(_ context.Context, _, _ string, _ *proto.ModelDiscoveryRequest) (*proto.ModelDiscoveryResult, error) {
|
||||
return nil, proxy.ErrModelDiscoveryUnavailable
|
||||
}
|
||||
|
||||
func (c *testProxyController) GetOIDCValidationConfig() proxy.OIDCValidationConfig {
|
||||
return proxy.OIDCValidationConfig{}
|
||||
}
|
||||
@@ -222,65 +218,6 @@ func TestSendServiceUpdateToCluster_DeleteNoToken(t *testing.T) {
|
||||
assert.Empty(t, msg2.AuthToken)
|
||||
}
|
||||
|
||||
func TestSendServiceUpdateToCluster_RejectsStaleClusterMembership(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := &ProxyServiceServer{
|
||||
tokenStore: NewOneTimeTokenStore(ctx, testCacheStore(t)),
|
||||
}
|
||||
controller := newTestProxyController()
|
||||
s.SetProxyController(controller)
|
||||
|
||||
ch := registerFakeProxy(s, "proxy-a", "new-cluster.example.com")
|
||||
require.NoError(t, controller.RegisterProxyToCluster(ctx, "old-cluster.example.com", "proxy-a"))
|
||||
|
||||
s.SendServiceUpdateToCluster(ctx, &proto.ProxyMapping{
|
||||
Type: proto.ProxyMappingUpdateType_UPDATE_TYPE_CREATED,
|
||||
Id: "agent-network-provider-1",
|
||||
AccountId: "account-1",
|
||||
Domain: "agent.example.com",
|
||||
Path: []*proto.PathMapping{
|
||||
{Path: "/", Target: "http://ollama.internal:11434/"},
|
||||
},
|
||||
}, "old-cluster.example.com")
|
||||
|
||||
assert.True(t, drainEmpty(ch), "stale membership must not route a mapping to a different live cluster")
|
||||
}
|
||||
|
||||
func TestDisconnectProxy_RemovesSupersededMembershipFromOldCluster(t *testing.T) {
|
||||
controller := newTestProxyController()
|
||||
server := &ProxyServiceServer{proxyController: controller}
|
||||
oldCtx, cancelOld := context.WithCancel(context.Background())
|
||||
newCtx, cancelNew := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancelOld)
|
||||
t.Cleanup(cancelNew)
|
||||
|
||||
oldConnection := &proxyConnection{
|
||||
proxyID: "proxy-a",
|
||||
sessionID: "old-session",
|
||||
address: "old-cluster.example.com",
|
||||
ctx: oldCtx,
|
||||
cancel: cancelOld,
|
||||
}
|
||||
newConnection := &proxyConnection{
|
||||
proxyID: "proxy-a",
|
||||
sessionID: "new-session",
|
||||
address: "new-cluster.example.com",
|
||||
ctx: newCtx,
|
||||
cancel: cancelNew,
|
||||
}
|
||||
require.NoError(t, controller.RegisterProxyToCluster(context.Background(), oldConnection.address, oldConnection.proxyID))
|
||||
require.NoError(t, controller.RegisterProxyToCluster(context.Background(), newConnection.address, newConnection.proxyID))
|
||||
server.connectedProxies.Store(newConnection.proxyID, newConnection)
|
||||
|
||||
server.disconnectProxy(oldConnection)
|
||||
|
||||
assert.Empty(t, controller.GetProxiesForCluster(oldConnection.address))
|
||||
assert.Equal(t, []string{"proxy-a"}, controller.GetProxiesForCluster(newConnection.address))
|
||||
current, ok := server.connectedProxies.Load(newConnection.proxyID)
|
||||
require.True(t, ok)
|
||||
assert.Same(t, newConnection, current)
|
||||
}
|
||||
|
||||
func TestSendServiceUpdate_UniqueTokensPerProxy(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
tokenStore := NewOneTimeTokenStore(ctx, testCacheStore(t))
|
||||
|
||||
@@ -3,20 +3,30 @@ package auth
|
||||
import (
|
||||
"context"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/sync/singleflight"
|
||||
|
||||
"github.com/netbirdio/netbird/proxy/internal/types"
|
||||
"github.com/netbirdio/netbird/shared/management/proto"
|
||||
)
|
||||
|
||||
// tunnelCacheTTL caps how long a positive ValidateTunnelPeer result is
|
||||
// reused before re-fetching from management. 5 minutes balances freshness
|
||||
// against management load on busy mesh networks.
|
||||
// tunnelCacheTTL is the default cap on how long a positive ValidateTunnelPeer
|
||||
// result is reused before re-fetching from management. 5 minutes balances
|
||||
// freshness against management load on busy mesh networks. Override it with
|
||||
// envTunnelCacheTTL when an account needs authorization changes to take effect
|
||||
// sooner (at the cost of more ValidateTunnelPeer RPCs).
|
||||
const tunnelCacheTTL = 300 * time.Second
|
||||
|
||||
// envTunnelCacheTTL overrides tunnelCacheTTL. The value is a Go duration string
|
||||
// (e.g. "30s", "2m"); an unset, unparseable, or non-positive value keeps the
|
||||
// default.
|
||||
const envTunnelCacheTTL = "NB_PROXY_TUNNEL_CACHE_TTL"
|
||||
|
||||
// tunnelCachePerAccount caps the number of cached identities per account.
|
||||
// Bounded eviction avoids memory growth in pathological cases (huge peer
|
||||
// roster, brief request bursts) while staying generous for normal use.
|
||||
@@ -60,16 +70,35 @@ type accountBucket struct {
|
||||
order []tunnelCacheKey
|
||||
}
|
||||
|
||||
// newTunnelValidationCache constructs a cache with default TTL and bounds.
|
||||
// newTunnelValidationCache constructs a cache with the configured TTL
|
||||
// (envTunnelCacheTTL override or default) and default bounds.
|
||||
func newTunnelValidationCache() *tunnelValidationCache {
|
||||
return &tunnelValidationCache{
|
||||
entries: make(map[types.AccountID]*accountBucket),
|
||||
ttl: tunnelCacheTTL,
|
||||
ttl: tunnelCacheTTLFromEnv(),
|
||||
maxSize: tunnelCachePerAccount,
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
// tunnelCacheTTLFromEnv returns the tunnel-cache TTL, honoring the
|
||||
// envTunnelCacheTTL override. The override must be a positive Go duration
|
||||
// string (e.g. "30s", "2m"); anything unset, unparseable, or non-positive
|
||||
// falls back to tunnelCacheTTL.
|
||||
func tunnelCacheTTLFromEnv() time.Duration {
|
||||
raw := strings.TrimSpace(os.Getenv(envTunnelCacheTTL))
|
||||
if raw == "" {
|
||||
return tunnelCacheTTL
|
||||
}
|
||||
d, err := time.ParseDuration(raw)
|
||||
if err != nil || d <= 0 {
|
||||
log.Warnf("ignoring invalid %s=%q (want a positive Go duration like 30s or 2m); using default %s",
|
||||
envTunnelCacheTTL, raw, tunnelCacheTTL)
|
||||
return tunnelCacheTTL
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// get returns a cached response for the key, or nil when missing or
|
||||
// expired. Expired entries are evicted lazily on read.
|
||||
func (c *tunnelValidationCache) get(key tunnelCacheKey) *proto.ValidateTunnelPeerResponse {
|
||||
|
||||
@@ -169,3 +169,32 @@ func TestTunnelCache_BoundedSizeEvictsOldest(t *testing.T) {
|
||||
assert.NotNil(t, cache.get(keys[1]), "second-newest must remain cached")
|
||||
assert.NotNil(t, cache.get(keys[2]), "newest must remain cached")
|
||||
}
|
||||
|
||||
func TestTunnelCacheTTLFromEnv(t *testing.T) {
|
||||
t.Run("unset uses default", func(t *testing.T) {
|
||||
t.Setenv(envTunnelCacheTTL, "")
|
||||
assert.Equal(t, tunnelCacheTTL, tunnelCacheTTLFromEnv())
|
||||
})
|
||||
t.Run("valid duration overrides", func(t *testing.T) {
|
||||
t.Setenv(envTunnelCacheTTL, "45s")
|
||||
assert.Equal(t, 45*time.Second, tunnelCacheTTLFromEnv())
|
||||
})
|
||||
t.Run("whitespace trimmed", func(t *testing.T) {
|
||||
t.Setenv(envTunnelCacheTTL, " 2m ")
|
||||
assert.Equal(t, 2*time.Minute, tunnelCacheTTLFromEnv())
|
||||
})
|
||||
t.Run("unparseable uses default", func(t *testing.T) {
|
||||
t.Setenv(envTunnelCacheTTL, "nonsense")
|
||||
assert.Equal(t, tunnelCacheTTL, tunnelCacheTTLFromEnv())
|
||||
})
|
||||
t.Run("non-positive uses default", func(t *testing.T) {
|
||||
t.Setenv(envTunnelCacheTTL, "0s")
|
||||
assert.Equal(t, tunnelCacheTTL, tunnelCacheTTLFromEnv())
|
||||
t.Setenv(envTunnelCacheTTL, "-30s")
|
||||
assert.Equal(t, tunnelCacheTTL, tunnelCacheTTLFromEnv())
|
||||
})
|
||||
t.Run("constructor honors override", func(t *testing.T) {
|
||||
t.Setenv(envTunnelCacheTTL, "90s")
|
||||
assert.Equal(t, 90*time.Second, newTunnelValidationCache().ttl)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -28,12 +28,12 @@ func TestDefaultTable_FirstPartyModelCoverage(t *testing.T) {
|
||||
"ministral-8b-latest", "mistral-embed",
|
||||
},
|
||||
"anthropic": {
|
||||
"claude-fable-5", "claude-opus-4-8", "claude-opus-4-7", "claude-opus-4-6",
|
||||
"claude-fable-5", "claude-opus-5", "claude-opus-4-8", "claude-opus-4-7", "claude-opus-4-6",
|
||||
"claude-opus-4-1", "claude-sonnet-4-6", "claude-sonnet-4-5", "claude-haiku-4-5",
|
||||
},
|
||||
// bedrock keys are the normalized ids the request parser emits.
|
||||
"bedrock": {
|
||||
"anthropic.claude-opus-4-8", "anthropic.claude-opus-4-7", "anthropic.claude-opus-4-6",
|
||||
"anthropic.claude-opus-5", "anthropic.claude-opus-4-8", "anthropic.claude-opus-4-7", "anthropic.claude-opus-4-6",
|
||||
"anthropic.claude-opus-4-1", "anthropic.claude-sonnet-4-6", "anthropic.claude-sonnet-4-5",
|
||||
"anthropic.claude-haiku-4-5", "meta.llama3-3-70b-instruct",
|
||||
"amazon.nova-pro", "amazon.nova-lite", "amazon.nova-micro", "amazon.nova-2-lite",
|
||||
|
||||
@@ -180,6 +180,11 @@ anthropic:
|
||||
output_per_1k: 0.050
|
||||
cache_read_per_1k: 0.001
|
||||
cache_creation_per_1k: 0.0125
|
||||
claude-opus-5:
|
||||
input_per_1k: 0.005
|
||||
output_per_1k: 0.025
|
||||
cache_read_per_1k: 0.0005
|
||||
cache_creation_per_1k: 0.00625
|
||||
claude-opus-4-8:
|
||||
input_per_1k: 0.005
|
||||
output_per_1k: 0.025
|
||||
@@ -236,6 +241,11 @@ bedrock:
|
||||
# eu.anthropic.claude-sonnet-4-5-20250929-v1:0 -> anthropic.claude-sonnet-4-5.
|
||||
# Anthropic-on-Bedrock keeps the additive cache buckets (read ≈0.1x input,
|
||||
# write ≈1.25x input); Nova / Llama report no cache, so cost is input+output.
|
||||
anthropic.claude-opus-5:
|
||||
input_per_1k: 0.005
|
||||
output_per_1k: 0.025
|
||||
cache_read_per_1k: 0.0005
|
||||
cache_creation_per_1k: 0.00625
|
||||
anthropic.claude-opus-4-8:
|
||||
input_per_1k: 0.005
|
||||
output_per_1k: 0.025
|
||||
|
||||
@@ -10,11 +10,15 @@ import (
|
||||
)
|
||||
|
||||
// Config is the JSON-decoded shape accepted by the factory. The
|
||||
// runtime path consumes the normalised allowlist; raw config is not
|
||||
// runtime path consumes the normalised allowlists; raw config is not
|
||||
// retained beyond construction.
|
||||
type Config struct {
|
||||
ModelAllowlist []string `json:"model_allowlist"`
|
||||
PromptCapture PromptCapture `json:"prompt_capture"`
|
||||
// ProviderAllowlists maps a resolved provider id (KeyLLMResolvedProviderID) to
|
||||
// its model allowlist. A provider present is restricted to those models; one
|
||||
// absent is unrestricted. Kept per-provider so one provider's list can't leak
|
||||
// onto another.
|
||||
ProviderAllowlists map[string][]string `json:"provider_allowlists,omitempty"`
|
||||
PromptCapture PromptCapture `json:"prompt_capture"`
|
||||
}
|
||||
|
||||
// PromptCapture toggles the optional prompt capture + redaction step
|
||||
@@ -54,21 +58,28 @@ func isEmptyJSON(raw []byte) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// normaliseConfig lowercases and trims allowlist entries so the runtime
|
||||
// match is case-insensitive. Empty entries are dropped.
|
||||
// normaliseConfig lowercases and trims allowlist entries for case-insensitive
|
||||
// matching; empty entries drop. A provider whose entries all drop keeps an empty
|
||||
// (non-nil) list — "deny every model" — distinct from an absent provider
|
||||
// (unrestricted).
|
||||
func normaliseConfig(cfg Config) Config {
|
||||
if len(cfg.ModelAllowlist) == 0 {
|
||||
if len(cfg.ProviderAllowlists) == 0 {
|
||||
cfg.ProviderAllowlists = nil
|
||||
return cfg
|
||||
}
|
||||
cleaned := make([]string, 0, len(cfg.ModelAllowlist))
|
||||
for _, entry := range cfg.ModelAllowlist {
|
||||
n := normaliseModel(entry)
|
||||
if n == "" {
|
||||
continue
|
||||
cleaned := make(map[string][]string, len(cfg.ProviderAllowlists))
|
||||
for provider, models := range cfg.ProviderAllowlists {
|
||||
list := make([]string, 0, len(models))
|
||||
for _, entry := range models {
|
||||
n := normaliseModel(entry)
|
||||
if n == "" {
|
||||
continue
|
||||
}
|
||||
list = append(list, n)
|
||||
}
|
||||
cleaned = append(cleaned, n)
|
||||
cleaned[provider] = list
|
||||
}
|
||||
cfg.ModelAllowlist = cleaned
|
||||
cfg.ProviderAllowlists = cleaned
|
||||
return cfg
|
||||
}
|
||||
|
||||
|
||||
@@ -83,8 +83,9 @@ func (m *Middleware) MutationsSupported() bool { return false }
|
||||
// prompt capture only affects the metadata emitted alongside an allow.
|
||||
func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middleware.Output, error) {
|
||||
model, modelPresent := lookupMetadata(in.Metadata, middleware.KeyLLMModel)
|
||||
providerID, _ := lookupMetadata(in.Metadata, middleware.KeyLLMResolvedProviderID)
|
||||
|
||||
if denial := m.evaluateAllowlist(model, modelPresent); denial != nil {
|
||||
if denial := m.evaluateAllowlist(providerID, model, modelPresent); denial != nil {
|
||||
return denial, nil
|
||||
}
|
||||
|
||||
@@ -110,20 +111,32 @@ func (m *Middleware) Invoke(_ context.Context, in *middleware.Input) (*middlewar
|
||||
// is a no-op.
|
||||
func (m *Middleware) Close() error { return nil }
|
||||
|
||||
// evaluateAllowlist returns a deny Output when the configured allowlist
|
||||
// rejects the model. A nil return means the request should proceed.
|
||||
func (m *Middleware) evaluateAllowlist(model string, modelPresent bool) *middleware.Output {
|
||||
if len(m.cfg.ModelAllowlist) == 0 {
|
||||
// evaluateAllowlist denies when the resolved provider's allowlist rejects the
|
||||
// model; nil means proceed. Scoped to the provider llm_router resolved, so an
|
||||
// unrestricted provider (absent from config) is never caught by another's list.
|
||||
func (m *Middleware) evaluateAllowlist(providerID, model string, modelPresent bool) *middleware.Output {
|
||||
if len(m.cfg.ProviderAllowlists) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Fail closed: with an allowlist configured, a request whose model the
|
||||
// upstream parser could not extract (absent or empty) must be denied rather
|
||||
// than allowed. This is what enforces the allowlist for URL/path-routed
|
||||
// providers (Bedrock, Vertex, ...) whose model lives outside the JSON body.
|
||||
// Restrictions exist but the resolved provider is unknown, so we can't tell
|
||||
// if this request targets a restricted provider — fail closed. llm_router
|
||||
// normally stamps the provider first, so this is a defensive guard.
|
||||
if providerID == "" {
|
||||
return denyModel("", denyCodeModelUnknown, denyMessageModelUnknown, denyReasonModelUnknown)
|
||||
}
|
||||
allowlist, restricted := m.cfg.ProviderAllowlists[providerID]
|
||||
if !restricted {
|
||||
// This provider has no allowlist (some authorising policy left it
|
||||
// unrestricted); management owns any per-policy/group decision.
|
||||
return nil
|
||||
}
|
||||
// Fail closed: with an allowlist in effect for this provider, a request whose
|
||||
// model the parser couldn't extract (absent/empty) is denied. This enforces
|
||||
// the allowlist for path-routed providers (Bedrock, Vertex) with no body model.
|
||||
if !modelPresent || normaliseModel(model) == "" {
|
||||
return denyModel("", denyCodeModelUnknown, denyMessageModelUnknown, denyReasonModelUnknown)
|
||||
}
|
||||
if m.modelInAllowlist(model) {
|
||||
if modelInAllowlist(allowlist, model) {
|
||||
return nil
|
||||
}
|
||||
return denyModel(model, denyCodeModel, denyMessageModel, denyReasonModel)
|
||||
@@ -151,14 +164,15 @@ func denyModel(model, code, message, reason string) *middleware.Output {
|
||||
}
|
||||
}
|
||||
|
||||
// modelInAllowlist reports whether the model matches any allowlist
|
||||
// entry under the case-insensitive, trim-tolerant comparison rule.
|
||||
func (m *Middleware) modelInAllowlist(model string) bool {
|
||||
// modelInAllowlist reports whether the model matches any entry in the supplied
|
||||
// (already-normalised) allowlist under the case-insensitive, trim-tolerant
|
||||
// comparison rule.
|
||||
func modelInAllowlist(allowlist []string, model string) bool {
|
||||
normalised := normaliseModel(model)
|
||||
if normalised == "" {
|
||||
return false
|
||||
}
|
||||
for _, allowed := range m.cfg.ModelAllowlist {
|
||||
for _, allowed := range allowlist {
|
||||
if allowed == normalised {
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -26,6 +26,25 @@ func newInput(meta ...middleware.KV) *middleware.Input {
|
||||
return &middleware.Input{Slot: middleware.SlotOnRequest, Metadata: meta}
|
||||
}
|
||||
|
||||
const (
|
||||
testProvider = "prov-1"
|
||||
otherProvider = "prov-2"
|
||||
)
|
||||
|
||||
// providerCfg builds a Config restricting testProvider to the given models.
|
||||
func providerCfg(models ...string) Config {
|
||||
return Config{ProviderAllowlists: map[string][]string{testProvider: models}}
|
||||
}
|
||||
|
||||
// newInputProvider builds an input that carries a resolved provider id (as
|
||||
// llm_router would stamp) plus any extra metadata.
|
||||
func newInputProvider(provider string, meta ...middleware.KV) *middleware.Input {
|
||||
all := make([]middleware.KV, 0, len(meta)+1)
|
||||
all = append(all, middleware.KV{Key: middleware.KeyLLMResolvedProviderID, Value: provider})
|
||||
all = append(all, meta...)
|
||||
return &middleware.Input{Slot: middleware.SlotOnRequest, Metadata: all}
|
||||
}
|
||||
|
||||
func TestMiddlewareIdentity(t *testing.T) {
|
||||
mw := New(Config{})
|
||||
assert.Equal(t, ID, mw.ID(), "middleware ID must be llm_guardrail")
|
||||
@@ -47,12 +66,12 @@ func TestMiddlewareIdentity(t *testing.T) {
|
||||
|
||||
func TestAllowlistEmptyAllowsAnyModel(t *testing.T) {
|
||||
mw := New(Config{})
|
||||
out, err := mw.Invoke(context.Background(), newInput(
|
||||
out, err := mw.Invoke(context.Background(), newInputProvider(testProvider,
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: "gpt-4o"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision, "empty allowlist must allow any model")
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision, "no provider allowlists must allow any model")
|
||||
v, ok := metaValue(t, out.Metadata, middleware.KeyLLMPolicyDecision)
|
||||
require.True(t, ok, "decision metadata must be emitted")
|
||||
assert.Equal(t, "allow", v, "decision must be allow")
|
||||
@@ -62,8 +81,8 @@ func TestAllowlistEmptyAllowsAnyModel(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAllowlistMatchAllows(t *testing.T) {
|
||||
mw := New(Config{ModelAllowlist: []string{"gpt-4o", "claude-opus-4"}})
|
||||
out, err := mw.Invoke(context.Background(), newInput(
|
||||
mw := New(providerCfg("gpt-4o", "claude-opus-4"))
|
||||
out, err := mw.Invoke(context.Background(), newInputProvider(testProvider,
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: "gpt-4o"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
@@ -71,8 +90,8 @@ func TestAllowlistMatchAllows(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAllowlistMissDenies(t *testing.T) {
|
||||
mw := New(Config{ModelAllowlist: []string{"gpt-4o"}})
|
||||
out, err := mw.Invoke(context.Background(), newInput(
|
||||
mw := New(providerCfg("gpt-4o"))
|
||||
out, err := mw.Invoke(context.Background(), newInputProvider(testProvider,
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: "claude-opus-4"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
@@ -91,10 +110,10 @@ func TestAllowlistMissDenies(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAllowlistCaseInsensitive(t *testing.T) {
|
||||
mw := New(Config{ModelAllowlist: []string{" GPT-4o ", "Claude-OPUS-4"}})
|
||||
mw := New(providerCfg(" GPT-4o ", "Claude-OPUS-4"))
|
||||
cases := []string{"gpt-4o", "GPT-4O", " claude-opus-4 "}
|
||||
for _, model := range cases {
|
||||
out, err := mw.Invoke(context.Background(), newInput(
|
||||
out, err := mw.Invoke(context.Background(), newInputProvider(testProvider,
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: model},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
@@ -103,14 +122,15 @@ func TestAllowlistCaseInsensitive(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAllowlistMissingModelKeyDenies(t *testing.T) {
|
||||
// Fail closed: with an allowlist configured, a request whose model the
|
||||
// parser could not extract (URL/path-routed providers such as Bedrock or
|
||||
// Vertex whose shape wasn't recognised) must be denied, not allowed.
|
||||
mw := New(Config{ModelAllowlist: []string{"gpt-4o"}})
|
||||
out, err := mw.Invoke(context.Background(), newInput())
|
||||
// Fail closed: with an allowlist in effect for the resolved provider, a
|
||||
// request whose model the parser could not extract (URL/path-routed
|
||||
// providers such as Bedrock or Vertex whose shape wasn't recognised) must be
|
||||
// denied, not allowed.
|
||||
mw := New(providerCfg("gpt-4o"))
|
||||
out, err := mw.Invoke(context.Background(), newInputProvider(testProvider))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.Equal(t, middleware.DecisionDeny, out.Decision, "absent model must be denied when an allowlist is set")
|
||||
assert.Equal(t, middleware.DecisionDeny, out.Decision, "absent model must be denied when the provider is restricted")
|
||||
assert.Equal(t, 403, out.DenyStatus, "deny status must be 403")
|
||||
require.NotNil(t, out.DenyReason, "deny reason must be populated")
|
||||
assert.Equal(t, "llm_policy.model_unknown", out.DenyReason.Code, "deny code must be model_unknown")
|
||||
@@ -122,26 +142,101 @@ func TestAllowlistMissingModelKeyDenies(t *testing.T) {
|
||||
|
||||
func TestAllowlistEmptyModelValueDenies(t *testing.T) {
|
||||
// A present-but-empty model is as undeterminable as an absent one.
|
||||
mw := New(Config{ModelAllowlist: []string{"gpt-4o"}})
|
||||
out, err := mw.Invoke(context.Background(), newInput(
|
||||
mw := New(providerCfg("gpt-4o"))
|
||||
out, err := mw.Invoke(context.Background(), newInputProvider(testProvider,
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: " "},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.Equal(t, middleware.DecisionDeny, out.Decision, "empty model must be denied when an allowlist is set")
|
||||
assert.Equal(t, middleware.DecisionDeny, out.Decision, "empty model must be denied when the provider is restricted")
|
||||
require.NotNil(t, out.DenyReason, "deny reason must be populated")
|
||||
assert.Equal(t, "llm_policy.model_unknown", out.DenyReason.Code, "deny code must be model_unknown")
|
||||
}
|
||||
|
||||
func TestAllowlistEmptyListAllowsMissingModel(t *testing.T) {
|
||||
// Without an allowlist there is nothing to enforce, so a missing model is
|
||||
// still allowed — the fail-closed rule only applies when a list is set.
|
||||
// Without any provider allowlists there is nothing to enforce, so a missing
|
||||
// model is still allowed — the fail-closed rule only applies when a
|
||||
// restriction is in effect.
|
||||
mw := New(Config{})
|
||||
out, err := mw.Invoke(context.Background(), newInput())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision, "no allowlist must allow even without a model")
|
||||
}
|
||||
|
||||
func TestUnrestrictedProviderAllowsAnyModel(t *testing.T) {
|
||||
// The request resolved to otherProvider, which has no allowlist, so its
|
||||
// traffic must not be caught by testProvider's restriction — the
|
||||
// cross-provider-leak / false-deny guard.
|
||||
mw := New(providerCfg("gpt-4o"))
|
||||
out, err := mw.Invoke(context.Background(), newInputProvider(otherProvider,
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: "claude-opus-4"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision, "an unrestricted provider must not inherit another provider's allowlist")
|
||||
}
|
||||
|
||||
func TestPerProviderAllowlistsAreIsolated(t *testing.T) {
|
||||
// gpt-4o is allowed only on testProvider; claude-opus-4 only on
|
||||
// otherProvider. A model allowlisted for one provider must not be usable on
|
||||
// the other — the fail-closed layer never unions allowlists across providers.
|
||||
mw := New(Config{ProviderAllowlists: map[string][]string{
|
||||
testProvider: {"gpt-4o"},
|
||||
otherProvider: {"claude-opus-4"},
|
||||
}})
|
||||
out, err := mw.Invoke(context.Background(), newInputProvider(testProvider,
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: "claude-opus-4"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, middleware.DecisionDeny, out.Decision, "claude-opus-4 is allowed only on otherProvider, not testProvider")
|
||||
require.NotNil(t, out.DenyReason)
|
||||
assert.Equal(t, "llm_policy.model_blocked", out.DenyReason.Code, "cross-provider model must be blocked, not model_unknown")
|
||||
}
|
||||
|
||||
func TestRestrictionsButNoResolvedProviderFailsClosed(t *testing.T) {
|
||||
// Restrictions exist for the account but the resolved provider id is absent,
|
||||
// so the request cannot be scoped to a provider. Fail closed rather than
|
||||
// wave it through.
|
||||
mw := New(providerCfg("gpt-4o"))
|
||||
out, err := mw.Invoke(context.Background(), newInput(
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: "gpt-4o"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.Equal(t, middleware.DecisionDeny, out.Decision, "missing resolved provider must fail closed when restrictions exist")
|
||||
require.NotNil(t, out.DenyReason)
|
||||
assert.Equal(t, "llm_policy.model_unknown", out.DenyReason.Code, "deny code must be model_unknown")
|
||||
}
|
||||
|
||||
func TestEnabledButEmptyAllowlistDeniesEveryModel(t *testing.T) {
|
||||
// An allowlist-enabled provider with zero models is distinct from an
|
||||
// unrestricted (absent) provider: it must deny every model.
|
||||
mw := New(providerCfg())
|
||||
out, err := mw.Invoke(context.Background(), newInputProvider(testProvider,
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: "gpt-4o"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.Equal(t, middleware.DecisionDeny, out.Decision, "an enabled-but-empty allowlist must deny every model")
|
||||
require.NotNil(t, out.DenyReason)
|
||||
assert.Equal(t, "llm_policy.model_blocked", out.DenyReason.Code, "deny code must be model_blocked, not model_unknown")
|
||||
}
|
||||
|
||||
func TestFactoryAllEmptyEntriesDenyEveryModel(t *testing.T) {
|
||||
// All the provider's entries are blank; they collapse to a non-nil empty
|
||||
// list (deny everything for that provider), not "no restriction".
|
||||
raw := []byte(`{"provider_allowlists":{"prov-1":[""," "]}}`)
|
||||
mw, err := Factory{}.New(raw)
|
||||
require.NoError(t, err)
|
||||
out, err := mw.Invoke(context.Background(), newInputProvider(testProvider,
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: "gpt-4o"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, out)
|
||||
assert.Equal(t, middleware.DecisionDeny, out.Decision, "all-blank allowlist entries must still restrict the provider")
|
||||
require.NotNil(t, out.DenyReason)
|
||||
assert.Equal(t, "llm_policy.model_blocked", out.DenyReason.Code, "deny code must be model_blocked")
|
||||
}
|
||||
|
||||
func TestPromptCaptureDisabledEmitsNoPrompt(t *testing.T) {
|
||||
mw := New(Config{})
|
||||
out, err := mw.Invoke(context.Background(), newInput(
|
||||
@@ -217,8 +312,8 @@ func TestFactoryAcceptsZeroConfigs(t *testing.T) {
|
||||
|
||||
func TestFactoryDecodesValidConfig(t *testing.T) {
|
||||
cfg := Config{
|
||||
ModelAllowlist: []string{"gpt-4o"},
|
||||
PromptCapture: PromptCapture{Enabled: true, RedactPii: true},
|
||||
ProviderAllowlists: map[string][]string{testProvider: {"gpt-4o"}},
|
||||
PromptCapture: PromptCapture{Enabled: true, RedactPii: true},
|
||||
}
|
||||
raw, err := json.Marshal(cfg)
|
||||
require.NoError(t, err, "marshalling test config must succeed")
|
||||
@@ -234,15 +329,15 @@ func TestFactoryRejectsMalformedJSON(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFactoryNormalisesAllowlist(t *testing.T) {
|
||||
raw := []byte(`{"model_allowlist":[" GPT-4o ","",""," Claude-3 "]}`)
|
||||
raw := []byte(`{"provider_allowlists":{"prov-1":[" GPT-4o ","",""," Claude-3 "]}}`)
|
||||
mw, err := Factory{}.New(raw)
|
||||
require.NoError(t, err)
|
||||
out, err := mw.Invoke(context.Background(), newInput(
|
||||
out, err := mw.Invoke(context.Background(), newInputProvider(testProvider,
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: "gpt-4o"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, middleware.DecisionAllow, out.Decision, "factory must lowercase + trim allowlist entries")
|
||||
out2, err := mw.Invoke(context.Background(), newInput(
|
||||
out2, err := mw.Invoke(context.Background(), newInputProvider(testProvider,
|
||||
middleware.KV{Key: middleware.KeyLLMModel, Value: "claude-3"},
|
||||
))
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -175,7 +175,7 @@ func denyFromManagement(resp *proto.CheckLLMPolicyLimitsResponse) *middleware.Ou
|
||||
DenyStatus: 403,
|
||||
DenyReason: &middleware.DenyReason{
|
||||
Code: code,
|
||||
Message: "LLM policy limit exceeded",
|
||||
Message: denyMessageForCode(code),
|
||||
},
|
||||
Metadata: []middleware.KV{
|
||||
{Key: middleware.KeyLLMPolicyDecision, Value: "deny"},
|
||||
@@ -184,6 +184,21 @@ func denyFromManagement(resp *proto.CheckLLMPolicyLimitsResponse) *middleware.Ou
|
||||
}
|
||||
}
|
||||
|
||||
// denyMessageForCode maps a management deny code to a public message.
|
||||
// Model-allowlist rejections get a model-specific message matching the
|
||||
// local guardrail; everything else keeps the generic quota wording. The
|
||||
// message stays generic so it never leaks internal quota detail.
|
||||
func denyMessageForCode(code string) string {
|
||||
switch code {
|
||||
case "llm_policy.model_blocked":
|
||||
return "model is not in the policy allowlist"
|
||||
case "llm_policy.model_unknown":
|
||||
return "request model could not be determined for the policy allowlist"
|
||||
default:
|
||||
return "LLM policy limit exceeded"
|
||||
}
|
||||
}
|
||||
|
||||
// lookupKV returns the value associated with key, or the empty
|
||||
// string when absent.
|
||||
func lookupKV(kvs []middleware.KV, key string) string {
|
||||
|
||||
@@ -115,6 +115,46 @@ func TestInvoke_DenyConvertsToProxyDeny(t *testing.T) {
|
||||
assert.NotContains(t, out.DenyReason.Message, "1000", "internal cap numbers must not reach the caller")
|
||||
}
|
||||
|
||||
// TestInvoke_ModelDenyMessages proves a model-allowlist rejection gets a
|
||||
// model-specific public message rather than the generic quota wording, so a
|
||||
// blocked or undetermined model reads consistently with the local guardrail.
|
||||
func TestInvoke_ModelDenyMessages(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
code string
|
||||
message string
|
||||
}{
|
||||
{"blocked", "llm_policy.model_blocked", "model is not in the policy allowlist"},
|
||||
{"unknown", "llm_policy.model_unknown", "request model could not be determined for the policy allowlist"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
mgmt := &fakeMgmt{
|
||||
checkResp: &proto.CheckLLMPolicyLimitsResponse{
|
||||
Decision: "deny",
|
||||
DenyCode: tc.code,
|
||||
},
|
||||
}
|
||||
m := New(mgmt, nil)
|
||||
|
||||
out := runInvoke(t, m, &middleware.Input{
|
||||
AccountID: "acc-1",
|
||||
UserGroups: []string{"grp-engineers"},
|
||||
Metadata: []middleware.KV{
|
||||
{Key: middleware.KeyLLMResolvedProviderID, Value: "prov-1"},
|
||||
{Key: middleware.KeyLLMModel, Value: "some-model"},
|
||||
},
|
||||
})
|
||||
|
||||
assert.Equal(t, middleware.DecisionDeny, out.Decision)
|
||||
require.NotNil(t, out.DenyReason, "deny envelope must carry a reason payload")
|
||||
assert.Equal(t, tc.code, out.DenyReason.Code, "canonical deny code surfaces to the caller")
|
||||
assert.Equal(t, tc.message, out.DenyReason.Message,
|
||||
"model denials must use a model-specific message, matching the local guardrail")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestInvoke_NoMgmtClientPassesThrough proves the partial-wiring
|
||||
// safety: a middleware constructed without a management client
|
||||
// allows every request without attribution. This makes a half-set-up
|
||||
|
||||
@@ -25,10 +25,17 @@ func runParserGuardrail(t *testing.T, url string, body []byte, allowlist []strin
|
||||
})
|
||||
require.NoError(t, err, "parser must not error")
|
||||
|
||||
guard := llm_guardrail.New(llm_guardrail.Config{ModelAllowlist: allowlist})
|
||||
const providerID = "prov-under-test"
|
||||
guard := llm_guardrail.New(llm_guardrail.Config{
|
||||
ProviderAllowlists: map[string][]string{providerID: allowlist},
|
||||
})
|
||||
// The real chain has llm_router stamp the resolved provider id before the
|
||||
// guardrail runs; the parser doesn't, so add it here so the guardrail can
|
||||
// scope the allowlist to this provider.
|
||||
meta := append([]middleware.KV{{Key: middleware.KeyLLMResolvedProviderID, Value: providerID}}, parsed.Metadata...)
|
||||
out, err := guard.Invoke(context.Background(), &middleware.Input{
|
||||
Slot: middleware.SlotOnRequest,
|
||||
Metadata: parsed.Metadata,
|
||||
Metadata: meta,
|
||||
})
|
||||
require.NoError(t, err, "guardrail must not error")
|
||||
require.NotNil(t, out, "guardrail must return an output")
|
||||
|
||||
@@ -1,337 +0,0 @@
|
||||
// Package modeldiscovery fetches and normalizes model catalogs from
|
||||
// Agent Network provider endpoints. Discovery always runs on the proxy so
|
||||
// it observes the same network path as inference traffic.
|
||||
package modeldiscovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/netbirdio/netbird/proxy/internal/roundtrip"
|
||||
)
|
||||
|
||||
const (
|
||||
SourceOpenAIV1Models = "openai_v1_models"
|
||||
SourceOllamaAPITags = "ollama_api_tags"
|
||||
|
||||
defaultTimeout = 5 * time.Second
|
||||
maxResponseBytes = 1 << 20 // 1 MiB, after HTTP decompression.
|
||||
maxUpstreamURLBytes = 4096
|
||||
maxHeaderNameBytes = 256
|
||||
maxHeaderValueBytes = 64 << 10
|
||||
maxModels = 500
|
||||
maxModelIDBytes = 512
|
||||
)
|
||||
|
||||
// Request contains the provider-owned values management resolved from the
|
||||
// persisted provider record. Callers must not populate these fields from a
|
||||
// dashboard-supplied URL or credential.
|
||||
type Request struct {
|
||||
UpstreamURL string
|
||||
AuthHeaderName string
|
||||
AuthHeaderValue string
|
||||
SkipTLSVerify bool
|
||||
AllowOllamaFallback bool
|
||||
}
|
||||
|
||||
// Model is the deliberately small response surface returned to management.
|
||||
// Arbitrary fields supplied by an upstream never cross the control channel.
|
||||
type Model struct {
|
||||
ID string
|
||||
Label string
|
||||
}
|
||||
|
||||
// Result is a normalized model catalog and the endpoint shape that supplied
|
||||
// it.
|
||||
type Result struct {
|
||||
Models []Model
|
||||
Source string
|
||||
}
|
||||
|
||||
// Discoverer owns the HTTP client used for provider probes.
|
||||
type Discoverer struct {
|
||||
client *http.Client
|
||||
timeout time.Duration
|
||||
bodyLimit int64
|
||||
}
|
||||
|
||||
// New constructs a direct-upstream discoverer. The transport is the same
|
||||
// host-network transport family used by Agent Network inference routes.
|
||||
func New(logger *log.Logger) *Discoverer {
|
||||
return newWithTransport(roundtrip.NewDirectOnly(logger))
|
||||
}
|
||||
|
||||
func newWithTransport(transport http.RoundTripper) *Discoverer {
|
||||
return &Discoverer{
|
||||
client: &http.Client{
|
||||
Transport: transport,
|
||||
// Redirects could move a credentialed request away from the
|
||||
// persisted provider origin. Discovery never follows them.
|
||||
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
},
|
||||
timeout: defaultTimeout,
|
||||
bodyLimit: maxResponseBytes,
|
||||
}
|
||||
}
|
||||
|
||||
// Discover queries the OpenAI-compatible model-list endpoint. Ollama's native
|
||||
// tags endpoint is attempted only when explicitly enabled and the primary
|
||||
// endpoint reports that the route does not exist.
|
||||
func (d *Discoverer) Discover(ctx context.Context, in Request) (Result, error) {
|
||||
if d == nil || d.client == nil {
|
||||
return Result{}, errors.New("model discovery client is unavailable")
|
||||
}
|
||||
if err := validateRequest(in); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
timeout := d.timeout
|
||||
if timeout <= 0 || timeout > defaultTimeout {
|
||||
timeout = defaultTimeout
|
||||
}
|
||||
probeCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
models, err := d.fetchOpenAIModels(probeCtx, in)
|
||||
if err == nil {
|
||||
return Result{Models: models, Source: SourceOpenAIV1Models}, nil
|
||||
}
|
||||
if !in.AllowOllamaFallback || !isMissingEndpoint(err) {
|
||||
return Result{}, err
|
||||
}
|
||||
|
||||
models, err = d.fetchOllamaTags(probeCtx, in)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return Result{Models: models, Source: SourceOllamaAPITags}, nil
|
||||
}
|
||||
|
||||
func validateRequest(in Request) error {
|
||||
rawURL := strings.TrimSpace(in.UpstreamURL)
|
||||
if rawURL == "" {
|
||||
return errors.New("model discovery upstream URL is required")
|
||||
}
|
||||
if len(rawURL) > maxUpstreamURLBytes {
|
||||
return errors.New("model discovery upstream URL is too long")
|
||||
}
|
||||
if len(in.AuthHeaderName) > maxHeaderNameBytes || len(in.AuthHeaderValue) > maxHeaderValueBytes {
|
||||
return errors.New("model discovery authentication header is too large")
|
||||
}
|
||||
if (in.AuthHeaderName == "") != (in.AuthHeaderValue == "") {
|
||||
return errors.New("model discovery authentication header is incomplete")
|
||||
}
|
||||
if strings.ContainsAny(in.AuthHeaderName, "\r\n") || strings.ContainsAny(in.AuthHeaderValue, "\r\n") {
|
||||
return errors.New("model discovery authentication header is invalid")
|
||||
}
|
||||
if in.AuthHeaderName != "" && !strings.EqualFold(in.AuthHeaderName, "Authorization") {
|
||||
return errors.New("model discovery authentication header is unsupported")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Discoverer) fetchOpenAIModels(ctx context.Context, in Request) ([]Model, error) {
|
||||
body, err := d.fetch(ctx, in, "v1/models")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var payload struct {
|
||||
Data *[]struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &payload); err != nil || payload.Data == nil {
|
||||
return nil, errors.New("upstream returned invalid OpenAI model-list JSON")
|
||||
}
|
||||
|
||||
ids := make([]string, 0, len(*payload.Data))
|
||||
for _, model := range *payload.Data {
|
||||
ids = append(ids, model.ID)
|
||||
}
|
||||
return normalize(ids)
|
||||
}
|
||||
|
||||
func (d *Discoverer) fetchOllamaTags(ctx context.Context, in Request) ([]Model, error) {
|
||||
body, err := d.fetch(ctx, in, "api/tags")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var payload struct {
|
||||
Models *[]struct {
|
||||
Name string `json:"name"`
|
||||
Model string `json:"model"`
|
||||
} `json:"models"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &payload); err != nil || payload.Models == nil {
|
||||
return nil, errors.New("upstream returned invalid Ollama tags JSON")
|
||||
}
|
||||
|
||||
ids := make([]string, 0, len(*payload.Models))
|
||||
for _, model := range *payload.Models {
|
||||
id := model.Model
|
||||
if strings.TrimSpace(id) == "" {
|
||||
id = model.Name
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return normalize(ids)
|
||||
}
|
||||
|
||||
func (d *Discoverer) fetch(ctx context.Context, in Request, endpointPath string) ([]byte, error) {
|
||||
endpoint, err := buildEndpointURL(in.UpstreamURL, endpointPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
reqCtx := roundtrip.WithDirectUpstream(ctx)
|
||||
if in.SkipTLSVerify {
|
||||
reqCtx = roundtrip.WithSkipTLSVerify(reqCtx)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, endpoint.String(), nil)
|
||||
if err != nil {
|
||||
return nil, errors.New("create model discovery request")
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if in.AuthHeaderName != "" {
|
||||
// Discovery is currently enabled only for Ollama-compatible providers.
|
||||
// Canonicalizing the sole catalog-owned credential header keeps the
|
||||
// control message from becoming a generic arbitrary-header primitive.
|
||||
req.Header.Set("Authorization", in.AuthHeaderValue)
|
||||
}
|
||||
|
||||
resp, err := d.client.Do(req)
|
||||
if err != nil {
|
||||
if errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
return nil, errors.New("model discovery timed out")
|
||||
}
|
||||
if errors.Is(ctx.Err(), context.Canceled) {
|
||||
return nil, context.Canceled
|
||||
}
|
||||
// Deliberately omit the underlying error: net/http errors include the
|
||||
// internal URL, which should not be reflected through the public API.
|
||||
return nil, errors.New("model discovery request failed")
|
||||
}
|
||||
defer func() {
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, &upstreamStatusError{statusCode: resp.StatusCode}
|
||||
}
|
||||
|
||||
limit := d.bodyLimit
|
||||
if limit <= 0 || limit > maxResponseBytes {
|
||||
limit = maxResponseBytes
|
||||
}
|
||||
if resp.ContentLength > limit {
|
||||
return nil, errors.New("model discovery response is too large")
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
|
||||
if err != nil {
|
||||
return nil, errors.New("read model discovery response")
|
||||
}
|
||||
if int64(len(body)) > limit {
|
||||
return nil, errors.New("model discovery response is too large")
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func buildEndpointURL(rawURL, endpointPath string) (*url.URL, error) {
|
||||
parsed, err := url.Parse(strings.TrimSpace(rawURL))
|
||||
if err != nil || parsed.Host == "" || parsed.Hostname() == "" || parsed.Opaque != "" {
|
||||
return nil, errors.New("model discovery upstream URL is invalid")
|
||||
}
|
||||
switch strings.ToLower(parsed.Scheme) {
|
||||
case "http":
|
||||
parsed.Scheme = "http"
|
||||
case "https":
|
||||
parsed.Scheme = "https"
|
||||
default:
|
||||
return nil, errors.New("model discovery upstream URL must use http or https")
|
||||
}
|
||||
if parsed.User != nil {
|
||||
return nil, errors.New("model discovery upstream URL must not contain credentials")
|
||||
}
|
||||
|
||||
// Match Agent Network routing semantics: the static discovery path is
|
||||
// appended to any persisted base path. Queries and fragments on a provider
|
||||
// URL are not forwarded to inference and are likewise excluded here.
|
||||
parsed.Path = "/" + strings.TrimPrefix(path.Join(parsed.Path, endpointPath), "/")
|
||||
parsed.RawPath = ""
|
||||
parsed.RawQuery = ""
|
||||
parsed.ForceQuery = false
|
||||
parsed.Fragment = ""
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func normalize(ids []string) ([]Model, error) {
|
||||
if len(ids) > maxModels {
|
||||
return nil, errors.New("upstream returned too many models")
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{}, len(ids))
|
||||
normalized := make([]string, 0, len(ids))
|
||||
for _, raw := range ids {
|
||||
id := strings.TrimSpace(raw)
|
||||
if !validModelID(id) {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
normalized = append(normalized, id)
|
||||
}
|
||||
sort.Strings(normalized)
|
||||
|
||||
models := make([]Model, 0, len(normalized))
|
||||
for _, id := range normalized {
|
||||
models = append(models, Model{ID: id, Label: id})
|
||||
}
|
||||
return models, nil
|
||||
}
|
||||
|
||||
func validModelID(id string) bool {
|
||||
if id == "" || len(id) > maxModelIDBytes || !utf8.ValidString(id) {
|
||||
return false
|
||||
}
|
||||
for _, r := range id {
|
||||
if unicode.IsControl(r) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type upstreamStatusError struct {
|
||||
statusCode int
|
||||
}
|
||||
|
||||
func (e *upstreamStatusError) Error() string {
|
||||
return fmt.Sprintf("upstream returned HTTP %d", e.statusCode)
|
||||
}
|
||||
|
||||
func isMissingEndpoint(err error) bool {
|
||||
var statusErr *upstreamStatusError
|
||||
if !errors.As(err, &statusErr) {
|
||||
return false
|
||||
}
|
||||
return statusErr.statusCode == http.StatusNotFound || statusErr.statusCode == http.StatusMethodNotAllowed
|
||||
}
|
||||
@@ -1,315 +0,0 @@
|
||||
package modeldiscovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDiscoverOpenAIModels(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, http.MethodGet, r.Method)
|
||||
assert.Equal(t, "/gateway/v1/models", r.URL.Path)
|
||||
assert.Empty(t, r.URL.RawQuery)
|
||||
assert.Equal(t, "application/json", r.Header.Get("Accept"))
|
||||
assert.Equal(t, "Bearer secret", r.Header.Get("Authorization"))
|
||||
_, _ = io.WriteString(w, `{
|
||||
"object": "list",
|
||||
"data": [
|
||||
{"id": " qwen2.5:latest ", "owned_by": "ignored"},
|
||||
{"id": "llama3.2:latest"},
|
||||
{"id": "llama3.2:latest"},
|
||||
{"id": ""},
|
||||
{"id": "bad\u0000id"}
|
||||
]
|
||||
}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
discoverer := newWithTransport(http.DefaultTransport)
|
||||
result, err := discoverer.Discover(context.Background(), Request{
|
||||
UpstreamURL: server.URL + "/gateway/?ignored=true#fragment",
|
||||
AuthHeaderName: "authorization",
|
||||
AuthHeaderValue: "Bearer secret",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, SourceOpenAIV1Models, result.Source)
|
||||
assert.Equal(t, []Model{
|
||||
{ID: "llama3.2:latest", Label: "llama3.2:latest"},
|
||||
{ID: "qwen2.5:latest", Label: "qwen2.5:latest"},
|
||||
}, result.Models)
|
||||
}
|
||||
|
||||
func TestDiscoverOllamaFallback(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var primaryCalls atomic.Int32
|
||||
var fallbackCalls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/v1/models":
|
||||
primaryCalls.Add(1)
|
||||
http.NotFound(w, r)
|
||||
case "/api/tags":
|
||||
fallbackCalls.Add(1)
|
||||
_, _ = io.WriteString(w, `{
|
||||
"models": [
|
||||
{"name": "ignored-name", "model": "gemma3:latest"},
|
||||
{"name": "llama3.2:latest", "model": ""}
|
||||
]
|
||||
}`)
|
||||
default:
|
||||
http.Error(w, "unexpected path", http.StatusInternalServerError)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
result, err := newWithTransport(http.DefaultTransport).Discover(context.Background(), Request{
|
||||
UpstreamURL: server.URL,
|
||||
AllowOllamaFallback: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int32(1), primaryCalls.Load())
|
||||
assert.Equal(t, int32(1), fallbackCalls.Load())
|
||||
assert.Equal(t, SourceOllamaAPITags, result.Source)
|
||||
assert.Equal(t, []Model{
|
||||
{ID: "gemma3:latest", Label: "gemma3:latest"},
|
||||
{ID: "llama3.2:latest", Label: "llama3.2:latest"},
|
||||
}, result.Models)
|
||||
}
|
||||
|
||||
func TestDiscoverDoesNotFallbackOnAuthenticationFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var fallbackCalls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/tags" {
|
||||
fallbackCalls.Add(1)
|
||||
}
|
||||
http.Error(w, "secret upstream body", http.StatusUnauthorized)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
_, err := newWithTransport(http.DefaultTransport).Discover(context.Background(), Request{
|
||||
UpstreamURL: server.URL,
|
||||
AllowOllamaFallback: true,
|
||||
})
|
||||
require.EqualError(t, err, "upstream returned HTTP 401")
|
||||
assert.Zero(t, fallbackCalls.Load())
|
||||
assert.NotContains(t, err.Error(), "secret upstream body")
|
||||
assert.NotContains(t, err.Error(), server.URL)
|
||||
}
|
||||
|
||||
func TestDiscoverDoesNotFollowRedirectsOrLeakAuth(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var redirectedCalls atomic.Int32
|
||||
redirected := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
redirectedCalls.Add(1)
|
||||
assert.Empty(t, r.Header.Get("Authorization"))
|
||||
_, _ = io.WriteString(w, `{"data":[]}`)
|
||||
}))
|
||||
defer redirected.Close()
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, redirected.URL+"/captured", http.StatusFound)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
_, err := newWithTransport(http.DefaultTransport).Discover(context.Background(), Request{
|
||||
UpstreamURL: server.URL,
|
||||
AuthHeaderName: "Authorization",
|
||||
AuthHeaderValue: "Bearer must-not-leak",
|
||||
})
|
||||
require.EqualError(t, err, "upstream returned HTTP 302")
|
||||
assert.Zero(t, redirectedCalls.Load())
|
||||
}
|
||||
|
||||
func TestDiscoverEnforcesResponseLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = io.WriteString(w, `{"data":[{"id":"llama3.2:latest"}]}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
discoverer := newWithTransport(http.DefaultTransport)
|
||||
discoverer.bodyLimit = 16
|
||||
_, err := discoverer.Discover(context.Background(), Request{UpstreamURL: server.URL})
|
||||
require.EqualError(t, err, "model discovery response is too large")
|
||||
}
|
||||
|
||||
func TestDiscoverEnforcesTimeout(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
case <-time.After(time.Second):
|
||||
_, _ = io.WriteString(w, `{"data":[]}`)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
discoverer := newWithTransport(http.DefaultTransport)
|
||||
discoverer.timeout = 30 * time.Millisecond
|
||||
_, err := discoverer.Discover(context.Background(), Request{UpstreamURL: server.URL})
|
||||
require.EqualError(t, err, "model discovery timed out")
|
||||
}
|
||||
|
||||
func TestDiscoverHonorsSkipTLSVerification(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = io.WriteString(w, `{"data":[]}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
discoverer := New(nil)
|
||||
_, err := discoverer.Discover(context.Background(), Request{UpstreamURL: server.URL})
|
||||
require.EqualError(t, err, "model discovery request failed")
|
||||
|
||||
result, err := discoverer.Discover(context.Background(), Request{
|
||||
UpstreamURL: server.URL,
|
||||
SkipTLSVerify: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, result.Models)
|
||||
}
|
||||
|
||||
func TestDiscoverRejectsUnsafeRequestValues(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
request Request
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "unsupported scheme",
|
||||
request: Request{UpstreamURL: "file:///etc/passwd"},
|
||||
wantErr: "model discovery upstream URL is invalid",
|
||||
},
|
||||
{
|
||||
name: "URL credentials",
|
||||
request: Request{UpstreamURL: "http://user:pass@example.com"},
|
||||
wantErr: "model discovery upstream URL must not contain credentials",
|
||||
},
|
||||
{
|
||||
name: "missing hostname",
|
||||
request: Request{UpstreamURL: "http://:11434"},
|
||||
wantErr: "model discovery upstream URL is invalid",
|
||||
},
|
||||
{
|
||||
name: "incomplete auth header",
|
||||
request: Request{
|
||||
UpstreamURL: "http://example.com",
|
||||
AuthHeaderName: "Authorization",
|
||||
},
|
||||
wantErr: "model discovery authentication header is incomplete",
|
||||
},
|
||||
{
|
||||
name: "header injection",
|
||||
request: Request{
|
||||
UpstreamURL: "http://example.com",
|
||||
AuthHeaderName: "Authorization",
|
||||
AuthHeaderValue: "Bearer safe\r\nX-Evil: yes",
|
||||
},
|
||||
wantErr: "model discovery authentication header is invalid",
|
||||
},
|
||||
{
|
||||
name: "unsupported auth header",
|
||||
request: Request{
|
||||
UpstreamURL: "http://example.com",
|
||||
AuthHeaderName: "X-API-Key",
|
||||
AuthHeaderValue: "secret",
|
||||
},
|
||||
wantErr: "model discovery authentication header is unsupported",
|
||||
},
|
||||
{
|
||||
name: "oversized URL",
|
||||
request: Request{
|
||||
UpstreamURL: "http://example.com/" + strings.Repeat("a", maxUpstreamURLBytes),
|
||||
},
|
||||
wantErr: "model discovery upstream URL is too long",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
_, err := newWithTransport(http.DefaultTransport).Discover(context.Background(), test.request)
|
||||
require.EqualError(t, err, test.wantErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverRejectsInvalidPayloads(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
status int
|
||||
body string
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "malformed JSON",
|
||||
status: http.StatusOK,
|
||||
body: `{"data":`,
|
||||
wantErr: "upstream returned invalid OpenAI model-list JSON",
|
||||
},
|
||||
{
|
||||
name: "missing data",
|
||||
status: http.StatusOK,
|
||||
body: `{}`,
|
||||
wantErr: "upstream returned invalid OpenAI model-list JSON",
|
||||
},
|
||||
{
|
||||
name: "too many models",
|
||||
status: http.StatusOK,
|
||||
body: modelListJSON(maxModels + 1),
|
||||
wantErr: "upstream returned too many models",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(test.status)
|
||||
_, _ = io.WriteString(w, test.body)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
_, err := newWithTransport(http.DefaultTransport).Discover(context.Background(), Request{
|
||||
UpstreamURL: server.URL,
|
||||
})
|
||||
require.EqualError(t, err, test.wantErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func modelListJSON(count int) string {
|
||||
var body strings.Builder
|
||||
body.WriteString(`{"data":[`)
|
||||
for i := range count {
|
||||
if i > 0 {
|
||||
body.WriteByte(',')
|
||||
}
|
||||
_, _ = fmt.Fprintf(&body, `{"id":"model-%d"}`, i)
|
||||
}
|
||||
body.WriteString(`]}`)
|
||||
return body.String()
|
||||
}
|
||||
@@ -271,10 +271,6 @@ func (c *testProxyController) SendServiceUpdateToCluster(_ context.Context, _ st
|
||||
// noop
|
||||
}
|
||||
|
||||
func (c *testProxyController) DiscoverModels(_ context.Context, _, _ string, _ *proto.ModelDiscoveryRequest) (*proto.ModelDiscoveryResult, error) {
|
||||
return nil, nbproxy.ErrModelDiscoveryUnavailable
|
||||
}
|
||||
|
||||
func (c *testProxyController) GetOIDCValidationConfig() nbproxy.OIDCValidationConfig {
|
||||
return nbproxy.OIDCValidationConfig{}
|
||||
}
|
||||
|
||||
@@ -1,349 +0,0 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/netbirdio/netbird/proxy/internal/crowdsec"
|
||||
"github.com/netbirdio/netbird/proxy/internal/modeldiscovery"
|
||||
"github.com/netbirdio/netbird/shared/management/proto"
|
||||
)
|
||||
|
||||
type stubModelDiscoverer struct {
|
||||
started chan modeldiscovery.Request
|
||||
release <-chan struct{}
|
||||
result modeldiscovery.Result
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *stubModelDiscoverer) Discover(ctx context.Context, request modeldiscovery.Request) (modeldiscovery.Result, error) {
|
||||
if s.started != nil {
|
||||
select {
|
||||
case s.started <- request:
|
||||
case <-ctx.Done():
|
||||
return modeldiscovery.Result{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
if s.release != nil {
|
||||
select {
|
||||
case <-s.release:
|
||||
case <-ctx.Done():
|
||||
return modeldiscovery.Result{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
return s.result, s.err
|
||||
}
|
||||
|
||||
type modelDiscoverySyncStream struct {
|
||||
grpc.ClientStream
|
||||
ctx context.Context
|
||||
recv chan *proto.SyncMappingsResponse
|
||||
sent chan *proto.SyncMappingsRequest
|
||||
sendWait time.Duration
|
||||
sending atomic.Int32
|
||||
overlap atomic.Bool
|
||||
}
|
||||
|
||||
func newModelDiscoverySyncStream(ctx context.Context) *modelDiscoverySyncStream {
|
||||
return &modelDiscoverySyncStream{
|
||||
ctx: ctx,
|
||||
recv: make(chan *proto.SyncMappingsResponse, 16),
|
||||
sent: make(chan *proto.SyncMappingsRequest, 16),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *modelDiscoverySyncStream) Send(message *proto.SyncMappingsRequest) error {
|
||||
if s.sending.Add(1) != 1 {
|
||||
s.overlap.Store(true)
|
||||
}
|
||||
defer s.sending.Add(-1)
|
||||
if s.sendWait > 0 {
|
||||
time.Sleep(s.sendWait)
|
||||
}
|
||||
select {
|
||||
case s.sent <- message:
|
||||
return nil
|
||||
case <-s.ctx.Done():
|
||||
return s.ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *modelDiscoverySyncStream) Recv() (*proto.SyncMappingsResponse, error) {
|
||||
select {
|
||||
case message, ok := <-s.recv:
|
||||
if !ok {
|
||||
return nil, io.EOF
|
||||
}
|
||||
return message, nil
|
||||
case <-s.ctx.Done():
|
||||
return nil, s.ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *modelDiscoverySyncStream) Context() context.Context {
|
||||
return s.ctx
|
||||
}
|
||||
|
||||
func completeModelDiscoveryInitialSync(t *testing.T, stream *modelDiscoverySyncStream) {
|
||||
t.Helper()
|
||||
stream.recv <- &proto.SyncMappingsResponse{InitialSyncComplete: true}
|
||||
select {
|
||||
case sent := <-stream.sent:
|
||||
require.NotNil(t, sent.GetAck())
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("initial snapshot was not acknowledged")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyCapabilitiesAdvertiseModelDiscovery(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := &Server{
|
||||
crowdsecRegistry: crowdsec.NewRegistry("", "", log.New().WithField("test", true)),
|
||||
}
|
||||
capabilities := server.proxyCapabilities()
|
||||
require.NotNil(t, capabilities.SupportsModelDiscovery)
|
||||
assert.True(t, capabilities.GetSupportsModelDiscovery())
|
||||
}
|
||||
|
||||
func TestExecuteModelDiscoveryMapsControlShape(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
discoverer := &stubModelDiscoverer{
|
||||
result: modeldiscovery.Result{
|
||||
Source: modeldiscovery.SourceOpenAIV1Models,
|
||||
Models: []modeldiscovery.Model{
|
||||
{ID: "llama3.2:latest", Label: "Llama 3.2"},
|
||||
},
|
||||
},
|
||||
}
|
||||
request := &proto.ModelDiscoveryRequest{
|
||||
RequestId: "request-1",
|
||||
UpstreamUrl: "http://ollama.internal:11434",
|
||||
AuthHeaderName: "Authorization",
|
||||
AuthHeaderValue: "Bearer secret",
|
||||
SkipTlsVerify: true,
|
||||
OllamaFallback: true,
|
||||
}
|
||||
|
||||
result := executeModelDiscovery(context.Background(), discoverer, request)
|
||||
assert.Equal(t, "request-1", result.GetRequestId())
|
||||
assert.Equal(t, modeldiscovery.SourceOpenAIV1Models, result.GetSource())
|
||||
require.Len(t, result.GetModels(), 1)
|
||||
assert.Equal(t, "llama3.2:latest", result.GetModels()[0].GetId())
|
||||
assert.Equal(t, "Llama 3.2", result.GetModels()[0].GetLabel())
|
||||
|
||||
discoverer.started = make(chan modeldiscovery.Request, 1)
|
||||
_ = executeModelDiscovery(context.Background(), discoverer, request)
|
||||
received := <-discoverer.started
|
||||
assert.Equal(t, request.GetUpstreamUrl(), received.UpstreamURL)
|
||||
assert.Equal(t, request.GetAuthHeaderName(), received.AuthHeaderName)
|
||||
assert.Equal(t, request.GetAuthHeaderValue(), received.AuthHeaderValue)
|
||||
assert.True(t, received.SkipTLSVerify)
|
||||
assert.True(t, received.AllowOllamaFallback)
|
||||
}
|
||||
|
||||
func TestExecuteModelDiscoveryReturnsSanitizedError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
result := executeModelDiscovery(context.Background(), &stubModelDiscoverer{
|
||||
err: errors.New("model discovery request failed"),
|
||||
}, &proto.ModelDiscoveryRequest{RequestId: "request-error"})
|
||||
assert.Equal(t, "request-error", result.GetRequestId())
|
||||
assert.Equal(t, "model discovery request failed", result.GetError())
|
||||
assert.Empty(t, result.GetModels())
|
||||
assert.Empty(t, result.GetSource())
|
||||
}
|
||||
|
||||
func TestHandleSyncMappingsStreamRunsDiscoveryOutOfBand(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
release := make(chan struct{})
|
||||
started := make(chan modeldiscovery.Request, 1)
|
||||
server := &Server{
|
||||
Logger: log.New(),
|
||||
routerReady: closedChan(),
|
||||
modelDiscoverer: &stubModelDiscoverer{
|
||||
started: started,
|
||||
release: release,
|
||||
result: modeldiscovery.Result{
|
||||
Source: modeldiscovery.SourceOpenAIV1Models,
|
||||
Models: []modeldiscovery.Model{{ID: "model-a", Label: "model-a"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
stream := newModelDiscoverySyncStream(ctx)
|
||||
stream.sendWait = 10 * time.Millisecond
|
||||
|
||||
done := make(chan error, 1)
|
||||
initialSyncDone := false
|
||||
go func() {
|
||||
done <- server.handleSyncMappingsStream(ctx, stream, &initialSyncDone, time.Time{})
|
||||
}()
|
||||
completeModelDiscoveryInitialSync(t, stream)
|
||||
|
||||
stream.recv <- &proto.SyncMappingsResponse{
|
||||
ModelDiscoveryRequest: &proto.ModelDiscoveryRequest{
|
||||
RequestId: "request-1",
|
||||
UpstreamUrl: "http://ollama.internal:11434",
|
||||
},
|
||||
}
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("model discovery did not start")
|
||||
}
|
||||
|
||||
// A normal mapping batch must still be acknowledged while the HTTP probe
|
||||
// is in flight.
|
||||
stream.recv <- &proto.SyncMappingsResponse{}
|
||||
select {
|
||||
case sent := <-stream.sent:
|
||||
assert.NotNil(t, sent.GetAck())
|
||||
assert.Nil(t, sent.GetModelDiscoveryResult())
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("mapping ack was blocked by model discovery")
|
||||
}
|
||||
|
||||
close(release)
|
||||
select {
|
||||
case sent := <-stream.sent:
|
||||
result := sent.GetModelDiscoveryResult()
|
||||
require.NotNil(t, result)
|
||||
assert.Equal(t, "request-1", result.GetRequestId())
|
||||
assert.Equal(t, modeldiscovery.SourceOpenAIV1Models, result.GetSource())
|
||||
assert.Nil(t, sent.GetAck())
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("model discovery result was not sent")
|
||||
}
|
||||
assert.False(t, stream.overlap.Load(), "acks and discovery results must use one serialized sender")
|
||||
|
||||
close(stream.recv)
|
||||
require.NoError(t, <-done)
|
||||
}
|
||||
|
||||
func TestHandleSyncMappingsStreamBoundsConcurrentDiscovery(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
release := make(chan struct{})
|
||||
started := make(chan modeldiscovery.Request, 8)
|
||||
server := &Server{
|
||||
Logger: log.New(),
|
||||
routerReady: closedChan(),
|
||||
modelDiscoverer: &stubModelDiscoverer{
|
||||
started: started,
|
||||
release: release,
|
||||
},
|
||||
}
|
||||
stream := newModelDiscoverySyncStream(ctx)
|
||||
|
||||
done := make(chan error, 1)
|
||||
initialSyncDone := false
|
||||
go func() {
|
||||
done <- server.handleSyncMappingsStream(ctx, stream, &initialSyncDone, time.Time{})
|
||||
}()
|
||||
completeModelDiscoveryInitialSync(t, stream)
|
||||
|
||||
for i := range 5 {
|
||||
stream.recv <- &proto.SyncMappingsResponse{
|
||||
ModelDiscoveryRequest: &proto.ModelDiscoveryRequest{
|
||||
RequestId: fmt.Sprintf("request-%d", i),
|
||||
UpstreamUrl: "http://ollama.internal:11434",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
for range 4 {
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("expected four concurrent model discoveries")
|
||||
}
|
||||
}
|
||||
select {
|
||||
case sent := <-stream.sent:
|
||||
result := sent.GetModelDiscoveryResult()
|
||||
require.NotNil(t, result)
|
||||
assert.Equal(t, "model discovery is busy", result.GetError())
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("fifth discovery did not fail fast")
|
||||
}
|
||||
|
||||
close(release)
|
||||
for range 4 {
|
||||
select {
|
||||
case sent := <-stream.sent:
|
||||
require.NotNil(t, sent.GetModelDiscoveryResult())
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("in-flight model discovery did not complete")
|
||||
}
|
||||
}
|
||||
close(stream.recv)
|
||||
require.NoError(t, <-done)
|
||||
}
|
||||
|
||||
func TestHandleSyncMappingsStreamRejectsMixedDiscoveryMessage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
server := &Server{
|
||||
Logger: log.New(),
|
||||
routerReady: closedChan(),
|
||||
modelDiscoverer: &stubModelDiscoverer{},
|
||||
}
|
||||
stream := newModelDiscoverySyncStream(ctx)
|
||||
stream.recv <- &proto.SyncMappingsResponse{
|
||||
Mapping: []*proto.ProxyMapping{{Id: "mapping-1"}},
|
||||
InitialSyncComplete: true,
|
||||
ModelDiscoveryRequest: &proto.ModelDiscoveryRequest{
|
||||
RequestId: "request-1",
|
||||
},
|
||||
}
|
||||
close(stream.recv)
|
||||
|
||||
initialSyncDone := true
|
||||
err := server.handleSyncMappingsStream(ctx, stream, &initialSyncDone, time.Time{})
|
||||
require.EqualError(t, err, "model discovery message must not include mapping data or set initial_sync_complete")
|
||||
}
|
||||
|
||||
func TestHandleSyncMappingsStreamRejectsDiscoveryBeforeInitialSnapshot(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
server := &Server{
|
||||
Logger: log.New(),
|
||||
routerReady: closedChan(),
|
||||
modelDiscoverer: &stubModelDiscoverer{},
|
||||
}
|
||||
stream := newModelDiscoverySyncStream(ctx)
|
||||
stream.recv <- &proto.SyncMappingsResponse{
|
||||
ModelDiscoveryRequest: &proto.ModelDiscoveryRequest{
|
||||
RequestId: "request-1",
|
||||
},
|
||||
}
|
||||
close(stream.recv)
|
||||
|
||||
initialSyncDone := true
|
||||
err := server.handleSyncMappingsStream(ctx, stream, &initialSyncDone, time.Time{})
|
||||
require.EqualError(t, err, "model discovery request received before initial sync completed")
|
||||
}
|
||||
136
proxy/server.go
136
proxy/server.go
@@ -57,7 +57,6 @@ import (
|
||||
proxymetrics "github.com/netbirdio/netbird/proxy/internal/metrics"
|
||||
"github.com/netbirdio/netbird/proxy/internal/middleware"
|
||||
mwbuiltin "github.com/netbirdio/netbird/proxy/internal/middleware/builtin"
|
||||
"github.com/netbirdio/netbird/proxy/internal/modeldiscovery"
|
||||
"github.com/netbirdio/netbird/proxy/internal/netutil"
|
||||
"github.com/netbirdio/netbird/proxy/internal/proxy"
|
||||
"github.com/netbirdio/netbird/proxy/internal/restrict"
|
||||
@@ -79,10 +78,6 @@ type portRouter struct {
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
type providerModelDiscoverer interface {
|
||||
Discover(context.Context, modeldiscovery.Request) (modeldiscovery.Result, error)
|
||||
}
|
||||
|
||||
type Server struct {
|
||||
ctx context.Context
|
||||
mgmtClient proto.ProxyServiceClient
|
||||
@@ -105,20 +100,16 @@ type Server struct {
|
||||
// middlewareRegistry is the source of registered middleware factories.
|
||||
// Concrete middlewares register themselves through init().
|
||||
middlewareRegistry *middleware.Registry
|
||||
// modelDiscoverer executes explicit provider model-list probes on the
|
||||
// proxy's host network. Lazily constructed for normal servers; injectable
|
||||
// in focused control-stream tests.
|
||||
modelDiscoverer providerModelDiscoverer
|
||||
mainRouter *nbtcp.Router
|
||||
mainPort uint16
|
||||
udpMu sync.Mutex
|
||||
udpRelays map[types.ServiceID]*udprelay.Relay
|
||||
udpRelayWg sync.WaitGroup
|
||||
portMu sync.RWMutex
|
||||
portRouters map[uint16]*portRouter
|
||||
svcPorts map[types.ServiceID][]uint16
|
||||
lastMappings map[types.ServiceID]*proto.ProxyMapping
|
||||
portRouterWg sync.WaitGroup
|
||||
mainRouter *nbtcp.Router
|
||||
mainPort uint16
|
||||
udpMu sync.Mutex
|
||||
udpRelays map[types.ServiceID]*udprelay.Relay
|
||||
udpRelayWg sync.WaitGroup
|
||||
portMu sync.RWMutex
|
||||
portRouters map[uint16]*portRouter
|
||||
svcPorts map[types.ServiceID][]uint16
|
||||
lastMappings map[types.ServiceID]*proto.ProxyMapping
|
||||
portRouterWg sync.WaitGroup
|
||||
|
||||
// hijackTracker tracks hijacked connections (e.g. WebSocket upgrades)
|
||||
// so they can be closed during graceful shutdown, since http.Server.Shutdown
|
||||
@@ -1286,16 +1277,12 @@ func (s *Server) proxyCapabilities() *proto.ProxyCapabilities {
|
||||
privateCapability := s.Private
|
||||
// Always true: this build enforces ProxyMapping.private via the auth middleware.
|
||||
supportsPrivateService := true
|
||||
// Model discovery is handled only on SyncMappings. Management also gates
|
||||
// dispatch on the live connection using this capability.
|
||||
supportsModelDiscovery := true
|
||||
return &proto.ProxyCapabilities{
|
||||
SupportsCustomPorts: &s.SupportsCustomPorts,
|
||||
RequireSubdomain: &s.RequireSubdomain,
|
||||
SupportsCrowdsec: &supportsCrowdSec,
|
||||
Private: &privateCapability,
|
||||
SupportsPrivateService: &supportsPrivateService,
|
||||
SupportsModelDiscovery: &supportsModelDiscovery,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1362,8 +1349,7 @@ func isSyncUnimplemented(err error) bool {
|
||||
// handleSyncMappingsStream consumes batches from a bidirectional SyncMappings
|
||||
// stream, sending an ack after each batch is fully processed. Management waits
|
||||
// for the ack before sending the next batch, providing application-level
|
||||
// back-pressure. Model discovery commands are out-of-band: they run with
|
||||
// bounded concurrency and return a correlated result instead of an ack.
|
||||
// back-pressure.
|
||||
func (s *Server) handleSyncMappingsStream(ctx context.Context, stream proto.ProxyService_SyncMappingsClient, initialSyncDone *bool, connectTime time.Time) error {
|
||||
select {
|
||||
case <-s.routerReady:
|
||||
@@ -1372,30 +1358,6 @@ func (s *Server) handleSyncMappingsStream(ctx context.Context, stream proto.Prox
|
||||
}
|
||||
|
||||
tracker := s.newSnapshotTracker(initialSyncDone, connectTime)
|
||||
initialSnapshotComplete := false
|
||||
discoverer := s.modelDiscoverer
|
||||
if discoverer == nil {
|
||||
discoverer = modeldiscovery.New(s.Logger)
|
||||
}
|
||||
|
||||
const maxConcurrentDiscoveries = 4
|
||||
discoverySlots := make(chan struct{}, maxConcurrentDiscoveries)
|
||||
discoveryCtx, cancelDiscoveries := context.WithCancel(ctx)
|
||||
var discoveryWG sync.WaitGroup
|
||||
var sendMu sync.Mutex
|
||||
defer func() {
|
||||
cancelDiscoveries()
|
||||
discoveryWG.Wait()
|
||||
}()
|
||||
|
||||
// gRPC permits one concurrent sender and one concurrent receiver, but not
|
||||
// multiple senders. Mapping acks and asynchronous discovery results share
|
||||
// this serialized send path.
|
||||
send := func(message *proto.SyncMappingsRequest) error {
|
||||
sendMu.Lock()
|
||||
defer sendMu.Unlock()
|
||||
return stream.Send(message)
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
@@ -1410,62 +1372,15 @@ func (s *Server) handleSyncMappingsStream(ctx context.Context, stream proto.Prox
|
||||
return fmt.Errorf("receive msg: %w", err)
|
||||
}
|
||||
|
||||
if discovery := msg.GetModelDiscoveryRequest(); discovery != nil {
|
||||
if len(msg.GetMapping()) != 0 || msg.GetInitialSyncComplete() {
|
||||
return errors.New("model discovery message must not include mapping data or set initial_sync_complete")
|
||||
}
|
||||
if !initialSnapshotComplete {
|
||||
return errors.New("model discovery request received before initial sync completed")
|
||||
}
|
||||
|
||||
select {
|
||||
case discoverySlots <- struct{}{}:
|
||||
discoveryWG.Add(1)
|
||||
go func() {
|
||||
defer discoveryWG.Done()
|
||||
defer func() { <-discoverySlots }()
|
||||
|
||||
result := executeModelDiscovery(discoveryCtx, discoverer, discovery)
|
||||
if discoveryCtx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err := send(&proto.SyncMappingsRequest{
|
||||
Msg: &proto.SyncMappingsRequest_ModelDiscoveryResult{
|
||||
ModelDiscoveryResult: result,
|
||||
},
|
||||
}); err != nil {
|
||||
s.Logger.WithError(err).Debug("failed to send model discovery result")
|
||||
}
|
||||
}()
|
||||
default:
|
||||
result := &proto.ModelDiscoveryResult{
|
||||
RequestId: discovery.GetRequestId(),
|
||||
Error: "model discovery is busy",
|
||||
}
|
||||
if err := send(&proto.SyncMappingsRequest{
|
||||
Msg: &proto.SyncMappingsRequest_ModelDiscoveryResult{
|
||||
ModelDiscoveryResult: result,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("send model discovery busy result: %w", err)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
batchStart := time.Now()
|
||||
s.Logger.Debug("Received mapping update, starting processing")
|
||||
if err := s.processMappingsGuarded(ctx, msg.GetMapping()); err != nil {
|
||||
return err
|
||||
}
|
||||
s.Logger.Debug("Processing mapping update completed")
|
||||
syncComplete := msg.GetInitialSyncComplete()
|
||||
tracker.recordBatch(ctx, s, msg.GetMapping(), syncComplete, batchStart)
|
||||
if syncComplete {
|
||||
initialSnapshotComplete = true
|
||||
}
|
||||
tracker.recordBatch(ctx, s, msg.GetMapping(), msg.GetInitialSyncComplete(), batchStart)
|
||||
|
||||
if err := send(&proto.SyncMappingsRequest{
|
||||
if err := stream.Send(&proto.SyncMappingsRequest{
|
||||
Msg: &proto.SyncMappingsRequest_Ack{Ack: &proto.SyncMappingsAck{}},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("send ack: %w", err)
|
||||
@@ -1474,31 +1389,6 @@ func (s *Server) handleSyncMappingsStream(ctx context.Context, stream proto.Prox
|
||||
}
|
||||
}
|
||||
|
||||
func executeModelDiscovery(ctx context.Context, discoverer providerModelDiscoverer, request *proto.ModelDiscoveryRequest) *proto.ModelDiscoveryResult {
|
||||
result := &proto.ModelDiscoveryResult{RequestId: request.GetRequestId()}
|
||||
discovered, err := discoverer.Discover(ctx, modeldiscovery.Request{
|
||||
UpstreamURL: request.GetUpstreamUrl(),
|
||||
AuthHeaderName: request.GetAuthHeaderName(),
|
||||
AuthHeaderValue: request.GetAuthHeaderValue(),
|
||||
SkipTLSVerify: request.GetSkipTlsVerify(),
|
||||
AllowOllamaFallback: request.GetOllamaFallback(),
|
||||
})
|
||||
if err != nil {
|
||||
result.Error = err.Error()
|
||||
return result
|
||||
}
|
||||
|
||||
result.Source = discovered.Source
|
||||
result.Models = make([]*proto.ModelDiscoveryModel, 0, len(discovered.Models))
|
||||
for _, model := range discovered.Models {
|
||||
result.Models = append(result.Models, &proto.ModelDiscoveryModel{
|
||||
Id: model.ID,
|
||||
Label: model.Label,
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// snapshotTracker accumulates service IDs during the initial snapshot and
|
||||
// finalises sync state when the complete flag arrives. Used by both
|
||||
// handleMappingStream and handleSyncMappingsStream so metric emission and
|
||||
|
||||
@@ -5138,9 +5138,6 @@ components:
|
||||
description: Models exposed through this endpoint, with the operator's per-1k input/output prices. Empty means all catalog models are allowed at catalog prices.
|
||||
items:
|
||||
$ref: '#/components/schemas/AgentNetworkProviderModel'
|
||||
has_api_key:
|
||||
type: boolean
|
||||
description: Whether an upstream API key is currently stored. The key itself is never returned.
|
||||
extra_values:
|
||||
type: object
|
||||
description: |
|
||||
@@ -5189,7 +5186,6 @@ components:
|
||||
- name
|
||||
- upstream_url
|
||||
- models
|
||||
- has_api_key
|
||||
- enabled
|
||||
- skip_tls_verification
|
||||
- metadata_disabled
|
||||
@@ -5216,8 +5212,7 @@ components:
|
||||
example: "eu.proxy.netbird.io"
|
||||
api_key:
|
||||
type: string
|
||||
description: |
|
||||
Upstream provider API key. Sealed at rest on the management server and never returned in responses. Whether a key is accepted or required is declared by the selected catalog provider's `auth_mode`. On update, omit this field to preserve the existing key or send an empty string to clear an optional key.
|
||||
description: Upstream provider API key. Sealed at rest on the management server and never returned in responses. Required on create; optional on update (omit to keep the existing key).
|
||||
example: "sk-..."
|
||||
models:
|
||||
type: array
|
||||
@@ -5280,51 +5275,6 @@ components:
|
||||
- id
|
||||
- input_per_1k
|
||||
- output_per_1k
|
||||
AgentNetworkDiscoveredModel:
|
||||
type: object
|
||||
description: A model reported by the provider's persisted upstream endpoint.
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
description: Exact model identifier accepted by the upstream endpoint.
|
||||
example: "llama3.2:latest"
|
||||
label:
|
||||
type: string
|
||||
description: Human-friendly label reported by the endpoint. Falls back to the model identifier.
|
||||
example: "llama3.2:latest"
|
||||
required:
|
||||
- id
|
||||
- label
|
||||
AgentNetworkModelDiscoveryResponse:
|
||||
type: object
|
||||
description: |
|
||||
Result of an explicit model-discovery probe executed by a connected
|
||||
proxy in the account's selected Agent Network cluster. Discovered
|
||||
models are not persisted until the provider is updated.
|
||||
properties:
|
||||
models:
|
||||
type: array
|
||||
description: Normalized, deduplicated models reported by the upstream.
|
||||
items:
|
||||
$ref: '#/components/schemas/AgentNetworkDiscoveredModel'
|
||||
source:
|
||||
type: string
|
||||
description: Upstream API shape used to obtain the result.
|
||||
enum: [openai_v1_models, ollama_api_tags]
|
||||
example: "openai_v1_models"
|
||||
proxy_cluster:
|
||||
type: string
|
||||
description: Proxy cluster from which the endpoint was queried.
|
||||
example: "eu.proxy.netbird.io"
|
||||
request_id:
|
||||
type: string
|
||||
description: Correlation identifier for the management-to-proxy probe.
|
||||
example: "b7ef7da0-78cc-4583-8c4f-55f2ce72015f"
|
||||
required:
|
||||
- models
|
||||
- source
|
||||
- proxy_cluster
|
||||
- request_id
|
||||
AgentNetworkCatalogModel:
|
||||
type: object
|
||||
properties:
|
||||
@@ -5375,11 +5325,6 @@ components:
|
||||
type: string
|
||||
description: Default upstream host suggested when adding a provider of this type.
|
||||
example: "api.openai.com"
|
||||
auth_mode:
|
||||
type: string
|
||||
description: Whether this provider requires, optionally accepts, or does not support an upstream API key.
|
||||
enum: [required, optional, none]
|
||||
example: "required"
|
||||
auth_header_template:
|
||||
type: string
|
||||
description: Template the proxy uses to inject the API key (the literal string ${API_KEY} is replaced at request time).
|
||||
@@ -5401,10 +5346,6 @@ components:
|
||||
"custom" — generic OpenAI-compatible self-hosted endpoint catch-all.
|
||||
enum: [provider, gateway, custom]
|
||||
example: "provider"
|
||||
supports_model_discovery:
|
||||
type: boolean
|
||||
description: Whether models can be loaded from a persisted endpoint through the selected proxy cluster.
|
||||
example: false
|
||||
extra_headers:
|
||||
type: array
|
||||
description: |
|
||||
@@ -5423,12 +5364,10 @@ components:
|
||||
- name
|
||||
- description
|
||||
- default_host
|
||||
- auth_mode
|
||||
- auth_header_template
|
||||
- default_content_type
|
||||
- brand_color
|
||||
- kind
|
||||
- supports_model_discovery
|
||||
- models
|
||||
AgentNetworkCatalogIdentityInjection:
|
||||
type: object
|
||||
@@ -14089,48 +14028,6 @@ paths:
|
||||
"$ref": "#/components/responses/not_found"
|
||||
'500':
|
||||
"$ref": "#/components/responses/internal_error"
|
||||
/api/agent-network/providers/{providerId}/discover-models:
|
||||
post:
|
||||
summary: Discover models from an Agent Network provider
|
||||
description: |
|
||||
Executes a bounded, read-only model-list probe from a connected proxy
|
||||
in the account's selected Agent Network cluster. The probe uses the
|
||||
provider's persisted endpoint, TLS setting, and stored credential.
|
||||
Returned models are not persisted by this operation.
|
||||
tags: [ Agent Network ]
|
||||
security:
|
||||
- BearerAuth: [ ]
|
||||
- TokenAuth: [ ]
|
||||
parameters:
|
||||
- in: path
|
||||
name: providerId
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
description: The unique identifier of a persisted Agent Network provider
|
||||
responses:
|
||||
'200':
|
||||
description: Models discovered successfully
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/AgentNetworkModelDiscoveryResponse'
|
||||
'400':
|
||||
"$ref": "#/components/responses/bad_request"
|
||||
'401':
|
||||
"$ref": "#/components/responses/requires_authentication"
|
||||
'403':
|
||||
"$ref": "#/components/responses/forbidden"
|
||||
'404':
|
||||
"$ref": "#/components/responses/not_found"
|
||||
'412':
|
||||
description: The provider cannot be queried or no capable proxy is connected
|
||||
content: { }
|
||||
'429':
|
||||
description: A discovery probe is already running or was requested too recently
|
||||
content: { }
|
||||
'500':
|
||||
"$ref": "#/components/responses/internal_error"
|
||||
/api/agent-network/policies:
|
||||
get:
|
||||
summary: List all Agent Network Policies
|
||||
|
||||
@@ -38,27 +38,6 @@ func (e AccessRestrictionsCrowdsecMode) Valid() bool {
|
||||
}
|
||||
}
|
||||
|
||||
// Defines values for AgentNetworkCatalogProviderAuthMode.
|
||||
const (
|
||||
AgentNetworkCatalogProviderAuthModeNone AgentNetworkCatalogProviderAuthMode = "none"
|
||||
AgentNetworkCatalogProviderAuthModeOptional AgentNetworkCatalogProviderAuthMode = "optional"
|
||||
AgentNetworkCatalogProviderAuthModeRequired AgentNetworkCatalogProviderAuthMode = "required"
|
||||
)
|
||||
|
||||
// Valid indicates whether the value is a known member of the AgentNetworkCatalogProviderAuthMode enum.
|
||||
func (e AgentNetworkCatalogProviderAuthMode) Valid() bool {
|
||||
switch e {
|
||||
case AgentNetworkCatalogProviderAuthModeNone:
|
||||
return true
|
||||
case AgentNetworkCatalogProviderAuthModeOptional:
|
||||
return true
|
||||
case AgentNetworkCatalogProviderAuthModeRequired:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Defines values for AgentNetworkCatalogProviderKind.
|
||||
const (
|
||||
AgentNetworkCatalogProviderKindCustom AgentNetworkCatalogProviderKind = "custom"
|
||||
@@ -98,24 +77,6 @@ func (e AgentNetworkConsumptionDimensionKind) Valid() bool {
|
||||
}
|
||||
}
|
||||
|
||||
// Defines values for AgentNetworkModelDiscoveryResponseSource.
|
||||
const (
|
||||
AgentNetworkModelDiscoveryResponseSourceOllamaApiTags AgentNetworkModelDiscoveryResponseSource = "ollama_api_tags"
|
||||
AgentNetworkModelDiscoveryResponseSourceOpenaiV1Models AgentNetworkModelDiscoveryResponseSource = "openai_v1_models"
|
||||
)
|
||||
|
||||
// Valid indicates whether the value is a known member of the AgentNetworkModelDiscoveryResponseSource enum.
|
||||
func (e AgentNetworkModelDiscoveryResponseSource) Valid() bool {
|
||||
switch e {
|
||||
case AgentNetworkModelDiscoveryResponseSourceOllamaApiTags:
|
||||
return true
|
||||
case AgentNetworkModelDiscoveryResponseSourceOpenaiV1Models:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Defines values for CreateAzureIntegrationRequestHost.
|
||||
const (
|
||||
CreateAzureIntegrationRequestHostMicrosoftCom CreateAzureIntegrationRequestHost = "microsoft.com"
|
||||
@@ -2077,9 +2038,6 @@ type AgentNetworkCatalogProvider struct {
|
||||
// AuthHeaderTemplate Template the proxy uses to inject the API key (the literal string ${API_KEY} is replaced at request time).
|
||||
AuthHeaderTemplate string `json:"auth_header_template"`
|
||||
|
||||
// AuthMode Whether this provider requires, optionally accepts, or does not support an upstream API key.
|
||||
AuthMode AgentNetworkCatalogProviderAuthMode `json:"auth_mode"`
|
||||
|
||||
// BrandColor Hex brand color used to render the provider badge in the dashboard.
|
||||
BrandColor string `json:"brand_color"`
|
||||
|
||||
@@ -2112,14 +2070,8 @@ type AgentNetworkCatalogProvider struct {
|
||||
|
||||
// Name Display name for the provider.
|
||||
Name string `json:"name"`
|
||||
|
||||
// SupportsModelDiscovery Whether models can be loaded from a persisted endpoint through the selected proxy cluster.
|
||||
SupportsModelDiscovery bool `json:"supports_model_discovery"`
|
||||
}
|
||||
|
||||
// AgentNetworkCatalogProviderAuthMode Whether this provider requires, optionally accepts, or does not support an upstream API key.
|
||||
type AgentNetworkCatalogProviderAuthMode string
|
||||
|
||||
// AgentNetworkCatalogProviderKind Presentation grouping for the provider Select on the dashboard.
|
||||
// "provider" — first-party vendor API (OpenAI, Anthropic, …); the upstream is the model itself.
|
||||
// "gateway" — routing/aggregation layer in front of multiple providers (LiteLLM, Portkey, …); typically pairs with NetBird identity stamping.
|
||||
@@ -2156,15 +2108,6 @@ type AgentNetworkConsumption struct {
|
||||
// AgentNetworkConsumptionDimensionKind Whether this row counts a single end user or a single source group across every member.
|
||||
type AgentNetworkConsumptionDimensionKind string
|
||||
|
||||
// AgentNetworkDiscoveredModel A model reported by the provider's persisted upstream endpoint.
|
||||
type AgentNetworkDiscoveredModel struct {
|
||||
// Id Exact model identifier accepted by the upstream endpoint.
|
||||
Id string `json:"id"`
|
||||
|
||||
// Label Human-friendly label reported by the endpoint. Falls back to the model identifier.
|
||||
Label string `json:"label"`
|
||||
}
|
||||
|
||||
// AgentNetworkGuardrail defines model for AgentNetworkGuardrail.
|
||||
type AgentNetworkGuardrail struct {
|
||||
// Checks Guardrail check parameters. Each entry has an `enabled` flag plus per-check configuration; disabled entries are inert.
|
||||
@@ -2212,26 +2155,6 @@ type AgentNetworkGuardrailRequest struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// AgentNetworkModelDiscoveryResponse Result of an explicit model-discovery probe executed by a connected
|
||||
// proxy in the account's selected Agent Network cluster. Discovered
|
||||
// models are not persisted until the provider is updated.
|
||||
type AgentNetworkModelDiscoveryResponse struct {
|
||||
// Models Normalized, deduplicated models reported by the upstream.
|
||||
Models []AgentNetworkDiscoveredModel `json:"models"`
|
||||
|
||||
// ProxyCluster Proxy cluster from which the endpoint was queried.
|
||||
ProxyCluster string `json:"proxy_cluster"`
|
||||
|
||||
// RequestId Correlation identifier for the management-to-proxy probe.
|
||||
RequestId string `json:"request_id"`
|
||||
|
||||
// Source Upstream API shape used to obtain the result.
|
||||
Source AgentNetworkModelDiscoveryResponseSource `json:"source"`
|
||||
}
|
||||
|
||||
// AgentNetworkModelDiscoveryResponseSource Upstream API shape used to obtain the result.
|
||||
type AgentNetworkModelDiscoveryResponseSource string
|
||||
|
||||
// AgentNetworkPolicy defines model for AgentNetworkPolicy.
|
||||
type AgentNetworkPolicy struct {
|
||||
// CreatedAt Timestamp when the policy was created.
|
||||
@@ -2337,9 +2260,6 @@ type AgentNetworkProvider struct {
|
||||
// ExtraValues Operator-typed values for catalog-declared extra headers. Keys are wire header names (e.g. `x-portkey-config`); values are the strings the proxy stamps on every upstream request to this provider. Catalog (AgentNetworkCatalogProvider.extra_headers) declares which keys are accepted; values not declared by the catalog are ignored at synth time. Empty / missing values mean no header stamped.
|
||||
ExtraValues *map[string]string `json:"extra_values,omitempty"`
|
||||
|
||||
// HasApiKey Whether an upstream API key is currently stored. The key itself is never returned.
|
||||
HasApiKey bool `json:"has_api_key"`
|
||||
|
||||
// Id Provider ID
|
||||
Id string `json:"id"`
|
||||
|
||||
@@ -2385,7 +2305,7 @@ type AgentNetworkProviderModel struct {
|
||||
|
||||
// AgentNetworkProviderRequest defines model for AgentNetworkProviderRequest.
|
||||
type AgentNetworkProviderRequest struct {
|
||||
// ApiKey Upstream provider API key. Sealed at rest on the management server and never returned in responses. Whether a key is accepted or required is declared by the selected catalog provider's `auth_mode`. On update, omit this field to preserve the existing key or send an empty string to clear an optional key.
|
||||
// ApiKey Upstream provider API key. Sealed at rest on the management server and never returned in responses. Required on create; optional on update (omit to keep the existing key).
|
||||
ApiKey *string `json:"api_key,omitempty"`
|
||||
|
||||
// BootstrapCluster Proxy cluster used to bootstrap the per-account agent-network endpoint when the first provider is created. Ignored on subsequent creates and on updates because the cluster is pinned on the account-level Settings row.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -73,10 +73,6 @@ message ProxyCapabilities {
|
||||
optional bool private = 4;
|
||||
// Whether the proxy enforces ProxyMapping.private (fails closed on ValidateTunnelPeer failure). Management MUST NOT stream private mappings to proxies that don't claim this.
|
||||
optional bool supports_private_service = 5;
|
||||
// Whether the proxy can execute correlated Agent Network model-discovery
|
||||
// requests delivered over SyncMappings. Management must not send discovery
|
||||
// requests to proxies that omit or disable this capability.
|
||||
optional bool supports_model_discovery = 6;
|
||||
}
|
||||
|
||||
// GetMappingUpdateRequest is sent to initialise a mapping stream.
|
||||
@@ -424,8 +420,6 @@ message SyncMappingsRequest {
|
||||
oneof msg {
|
||||
SyncMappingsInit init = 1;
|
||||
SyncMappingsAck ack = 2;
|
||||
// Correlated response to a ModelDiscoveryRequest received from management.
|
||||
ModelDiscoveryResult model_discovery_result = 3;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -449,9 +443,6 @@ message SyncMappingsResponse {
|
||||
repeated ProxyMapping mapping = 1;
|
||||
// initial_sync_complete is set on the last message of the initial snapshot.
|
||||
bool initial_sync_complete = 2;
|
||||
// An out-of-band model-discovery operation. This is sent only after the
|
||||
// initial snapshot and only to proxies advertising supports_model_discovery.
|
||||
ModelDiscoveryRequest model_discovery_request = 3;
|
||||
}
|
||||
|
||||
// CheckLLMPolicyLimitsRequest carries the resolved caller identity and the
|
||||
@@ -510,33 +501,3 @@ message RecordLLMUsageRequest {
|
||||
message RecordLLMUsageResponse {
|
||||
}
|
||||
|
||||
// ModelDiscoveryRequest asks a proxy to list models from a persisted provider
|
||||
// endpoint using the proxy host's network path. request_id is assigned by
|
||||
// management and echoed verbatim in ModelDiscoveryResult.
|
||||
message ModelDiscoveryRequest {
|
||||
string request_id = 1;
|
||||
string upstream_url = 2;
|
||||
string auth_header_name = 3;
|
||||
string auth_header_value = 4;
|
||||
bool skip_tls_verify = 5;
|
||||
// When true, the proxy may fall back from the OpenAI-compatible
|
||||
// GET /v1/models endpoint to Ollama's native GET /api/tags endpoint.
|
||||
bool ollama_fallback = 6;
|
||||
}
|
||||
|
||||
// ModelDiscoveryModel is the normalized model shape returned to management.
|
||||
// Arbitrary upstream fields never cross the proxy control channel.
|
||||
message ModelDiscoveryModel {
|
||||
string id = 1;
|
||||
string label = 2;
|
||||
}
|
||||
|
||||
// ModelDiscoveryResult completes one ModelDiscoveryRequest. error is empty on
|
||||
// success and contains only a sanitized diagnostic on failure.
|
||||
message ModelDiscoveryResult {
|
||||
string request_id = 1;
|
||||
repeated ModelDiscoveryModel models = 2;
|
||||
// Stable source label, currently openai_v1_models or ollama_api_tags.
|
||||
string source = 3;
|
||||
string error = 4;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user