diff --git a/e2e/agentnetwork/chat_test.go b/e2e/agentnetwork/chat_test.go index 9fc6e7ca3..6cd23c34e 100644 --- a/e2e/agentnetwork/chat_test.go +++ b/e2e/agentnetwork/chat_test.go @@ -96,6 +96,25 @@ func availableProviders() []providerCase { return ps } +// providerRequest builds a create request for a matrix provider: enabled, with +// a uniquely-priced model for body-routed providers and none for the +// path-routed Vertex (whose model lives in the request path). +func providerRequest(pc providerCase) api.AgentNetworkProviderRequest { + req := api.AgentNetworkProviderRequest{ + Name: pc.name, + ProviderId: pc.catalogID, + UpstreamUrl: pc.upstream, + ApiKey: &pc.apiKey, + Enabled: ptr(true), + } + if pc.kind != harness.WireVertex { + req.Models = &[]api.AgentNetworkProviderModel{ + {Id: pc.model, InputPer1k: 0.001, OutputPer1k: 0.002}, + } + } + return req +} + // TestProvidersMatrix is Pillar 3: it provisions every available provider (all // enabled, each with a unique model so routing stays unambiguous), runs proxy + // client once, and drives the same live chat-completion scenario through each @@ -134,20 +153,7 @@ func TestProvidersMatrix(t *testing.T) { // cluster. ids := make([]string, 0, len(matrix)) for i, pc := range matrix { - req := api.AgentNetworkProviderRequest{ - Name: pc.name, - ProviderId: pc.catalogID, - UpstreamUrl: pc.upstream, - ApiKey: &pc.apiKey, - Enabled: ptr(true), - } - // Vertex is path-routed (model lives in the rawPredict path), so it carries - // no models array; body-model providers list a unique model for routing. - if pc.kind != harness.WireVertex { - req.Models = &[]api.AgentNetworkProviderModel{ - {Id: pc.model, InputPer1k: 0.001, OutputPer1k: 0.002}, - } - } + req := providerRequest(pc) if i == 0 { req.BootstrapCluster = ptr(harness.AgentNetworkCluster) } @@ -164,6 +170,17 @@ func TestProvidersMatrix(t *testing.T) { Enabled: &enabled, SourceGroups: []string{grp.Id}, DestinationProviderIds: ids, + // Token limit at the 60s window floor with caps far above the few hundred + // tokens this suite drives, so it never blocks traffic but switches on + // usage metering, which is what makes consumption rows get recorded. + Limits: &api.AgentNetworkPolicyLimits{ + TokenLimit: api.AgentNetworkPolicyTokenLimit{ + Enabled: true, + GroupCap: 10_000_000, + UserCap: 10_000_000, + WindowSeconds: 60, + }, + }, }) require.NoError(t, err, "create policy") t.Cleanup(func() { _ = srv.DeletePolicy(context.Background(), pol.Id) }) @@ -224,4 +241,22 @@ func TestProvidersMatrix(t *testing.T) { }, 30*time.Second, 2*time.Second, "an access-log row should be ingested for %s", pc.name) }) } + + // Metering: the policy's uncapped token limit switches on usage recording, + // so the live traffic just driven must surface as consumption rows with + // positive token counts. Consumption is account-scoped (keyed by source + // group / user and time window, not per provider), and ingest is async, so + // poll for any row that has booked tokens. + require.Eventually(t, func() bool { + rows, lerr := srv.ListConsumption(ctx) + if lerr != nil { + return false + } + for _, r := range rows { + if r.TokensInput > 0 && r.TokensOutput > 0 { + return true + } + } + return false + }, 60*time.Second, 3*time.Second, "consumption must be recorded with positive token counts after live traffic") } diff --git a/e2e/agentnetwork/management_test.go b/e2e/agentnetwork/management_test.go index e87a6b678..cfd03f63c 100644 --- a/e2e/agentnetwork/management_test.go +++ b/e2e/agentnetwork/management_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/netbirdio/netbird/e2e/harness" "github.com/netbirdio/netbird/shared/management/client/rest" "github.com/netbirdio/netbird/shared/management/http/api" ) @@ -40,32 +41,67 @@ func requireClientError(t *testing.T, err error) { assert.Less(t, apiErr.StatusCode, 500, "expected a 4xx status") } -// TestProviderLifecycle covers create → get → list → delete → 404. +// TestProviderLifecycle covers create → get → list → delete → 404 for every +// available real provider catalog (and a synthetic OpenAI provider when no +// provider keys are set), so each catalog's create and field round-trip is +// exercised. Create is offline — no upstream call — so this stays fast and +// burns no provider quota. func TestProviderLifecycle(t *testing.T) { ctx := context.Background() - prov := newProvider(t, ctx, "Provider Lifecycle") - assert.NotEmpty(t, prov.Id, "created provider must have an id") - assert.Equal(t, "openai_api", prov.ProviderId) - - got, err := srv.GetProvider(ctx, prov.Id) - require.NoError(t, err, "get provider") - assert.Equal(t, prov.Id, got.Id) - - list, err := srv.ListProviders(ctx) - require.NoError(t, err, "list providers") - var ids []string - for _, p := range list { - ids = append(ids, p.Id) + cases := availableProviders() + if len(cases) == 0 { + cases = []providerCase{{ + name: "openai", catalogID: "openai_api", upstream: "https://api.openai.com", + apiKey: "sk-dummy-e2e-key", model: "gpt-4o-mini", kind: harness.WireChat, + }} } - assert.Contains(t, ids, prov.Id, "created provider must appear in the list") - require.NoError(t, srv.DeleteProvider(ctx, prov.Id), "delete provider") - _, err = srv.GetProvider(ctx, prov.Id) - requireClientError(t, err) + for i, pc := range cases { + i, pc := i, pc + t.Run(pc.name, func(t *testing.T) { + req := providerRequest(pc) + req.Name = "lc-" + pc.name + // Bootstrap the cluster on the first create in case the matrix has + // not run (e.g. no provider keys → settings not yet bootstrapped). + if i == 0 { + req.BootstrapCluster = ptr(harness.AgentNetworkCluster) + } + + prov, err := srv.CreateProvider(ctx, req) + require.NoError(t, err, "create %s provider", pc.name) + t.Cleanup(func() { _ = srv.DeleteProvider(context.Background(), prov.Id) }) + + assert.NotEmpty(t, prov.Id, "created provider must have an id") + assert.Equal(t, pc.catalogID, prov.ProviderId, "catalog id must round-trip") + assert.Equal(t, req.Name, prov.Name, "name must round-trip") + assert.Equal(t, pc.upstream, prov.UpstreamUrl, "upstream must round-trip") + + got, err := srv.GetProvider(ctx, prov.Id) + require.NoError(t, err, "get provider") + assert.Equal(t, prov.Id, got.Id) + + list, err := srv.ListProviders(ctx) + require.NoError(t, err, "list providers") + var ids []string + for _, p := range list { + ids = append(ids, p.Id) + } + assert.Contains(t, ids, prov.Id, "created provider must appear in the list") + + require.NoError(t, srv.DeleteProvider(ctx, prov.Id), "delete provider") + _, err = srv.GetProvider(ctx, prov.Id) + requireClientError(t, err) + }) + } } -// TestProviderValidation rejects a missing API key and an unknown catalog id. +// TestProviderValidation exercises the create-time validation rules. These are +// uniform across catalogs (no per-provider required-field rules exist: a +// catalog-specific malformed value such as a Vertex key without the keyfile:: +// prefix is accepted at create and only fails at the proxy), so the cases here +// are catalog-agnostic: missing API key, unknown catalog id, an invalid upstream +// URL, and a blank name. func TestProviderValidation(t *testing.T) { ctx := context.Background() @@ -83,6 +119,22 @@ func TestProviderValidation(t *testing.T) { ApiKey: ptr("sk-dummy"), }) requireClientError(t, err) + + _, err = srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ + Name: "Bad Upstream", + ProviderId: "openai_api", + UpstreamUrl: "not-a-url", + ApiKey: ptr("sk-dummy"), + }) + requireClientError(t, err) + + _, err = srv.CreateProvider(ctx, api.AgentNetworkProviderRequest{ + Name: " ", + ProviderId: "openai_api", + UpstreamUrl: "https://api.openai.com", + ApiKey: ptr("sk-dummy"), + }) + requireClientError(t, err) } // TestSettingsRoundTrip flips the collection toggles and confirms cluster /