[agent-network] Management: store, manager, synthesizer, policy engine, provider catalog, HTTP/gRPC API

Adds the account-scoped agent-network module: provider/policy/budget CRUD and
store, the reverse-proxy service synthesizer, policy selection + limit
enforcement, the provider catalog (incl. Vertex AI and AWS Bedrock entries),
and the management HTTP + proxy gRPC surfaces.

management/internals/modules/reverseproxy/accesslogs/accesslogentry.go

@@ -39,6 +39,10 @@ type AccessLogEntry struct {
 	BytesDownload   int64             `gorm:"index"`
 	Protocol        AccessLogProtocol `gorm:"index"`
 	Metadata        map[string]string `gorm:"serializer:json"`
+	// AgentNetwork marks the entry as emitted by a synthesised agent-network
+	// service. Sourced from proto.AccessLog.AgentNetwork the proxy stamps
+	// before shipping. Indexed so the agent-network log surface filters cheaply.
+	AgentNetwork bool `gorm:"index"`
 }
 
 // FromProto creates an AccessLogEntry from a proto.AccessLog
@@ -58,6 +62,7 @@ func (a *AccessLogEntry) FromProto(serviceLog *proto.AccessLog) {
 	a.BytesDownload = serviceLog.GetBytesDownload()
 	a.Protocol = AccessLogProtocol(serviceLog.GetProtocol())
 	a.Metadata = maps.Clone(serviceLog.GetMetadata())
+	a.AgentNetwork = serviceLog.GetAgentNetwork()
 
 	if sourceIP := serviceLog.GetSourceIp(); sourceIP != "" {
 		if addr, err := netip.ParseAddr(sourceIP); err == nil {

management/internals/modules/reverseproxy/accesslogs/manager/manager.go

@@ -2,12 +2,15 @@ package manager
 
 import (
 	"context"
+	"math"
+	"strconv"
 	"strings"
 	"time"
 
 	log "github.com/sirupsen/logrus"
 
 	"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
+	agentNetworkTypes "github.com/netbirdio/netbird/management/server/agentnetwork/types"
 	"github.com/netbirdio/netbird/management/server/geolocation"
 	"github.com/netbirdio/netbird/management/server/permissions"
 	"github.com/netbirdio/netbird/management/server/permissions/modules"
@@ -16,6 +19,28 @@ import (
 	"github.com/netbirdio/netbird/shared/management/status"
 )
 
+// Metadata keys the proxy stamps on agent-network access-log entries. These
+// mirror the constants in proxy/internal/middleware/keys.go and form the wire
+// contract between the proxy and management; management flattens them into
+// queryable columns. Keep in sync with the proxy side.
+const (
+	metaKeyProvider           = "llm.provider"
+	metaKeyModel              = "llm.model"
+	metaKeyResolvedProviderID = "llm.resolved_provider_id"
+	metaKeySelectedPolicyID   = "llm.selected_policy_id"
+	metaKeyPolicyDecision     = "llm_policy.decision"
+	metaKeyPolicyReason       = "llm_policy.reason"
+	metaKeyInputTokens        = "llm.input_tokens"  //nolint:gosec // metadata key name, not a credential
+	metaKeyOutputTokens       = "llm.output_tokens" //nolint:gosec // metadata key name, not a credential
+	metaKeyTotalTokens        = "llm.total_tokens"  //nolint:gosec // metadata key name, not a credential
+	metaKeyCostUSDTotal       = "cost.usd_total"
+	metaKeyStream             = "llm.stream"
+	metaKeySessionID          = "llm.session_id"
+	metaKeyAuthorisingGroups  = "llm.authorising_groups"
+	metaKeyRequestPrompt      = "llm.request_prompt"
+	metaKeyResponseCompletion = "llm.response_completion"
+)
+
 type managerImpl struct {
 	store              store.Store
 	permissionsManager permissions.Manager
@@ -31,8 +56,14 @@ func NewManager(store store.Store, permissionsManager permissions.Manager, geo g
 	}
 }
 
-// SaveAccessLog saves an access log entry to the database after enriching it
+// SaveAccessLog saves an access log entry to the database after enriching it.
+// Agent-network entries are flattened into their own dedicated table (queryable
+// LLM columns + group child rows) instead of the shared reverse-proxy table.
 func (m *managerImpl) SaveAccessLog(ctx context.Context, logEntry *accesslogs.AccessLogEntry) error {
+	if logEntry.AgentNetwork {
+		return m.saveAgentNetworkAccessLog(ctx, logEntry)
+	}
+
 	if m.geo != nil && logEntry.GeoLocation.ConnectionIP != nil {
 		location, err := m.geo.Lookup(logEntry.GeoLocation.ConnectionIP)
 		if err != nil {
@@ -61,6 +92,184 @@ func (m *managerImpl) SaveAccessLog(ctx context.Context, logEntry *accesslogs.Ac
 	return nil
 }
 
+// saveAgentNetworkAccessLog flattens the metadata-bearing access-log entry and
+// persists it in two parts:
+//
+//   - The stripped usage record is written unconditionally — usage/cost is
+//     collected on every request regardless of the account's log-collection
+//     toggle (the proxy ships a usage-only entry when logging is disabled).
+//   - The full access-log row (with request detail + prompt) is written only
+//     when the account's EnableLogCollection setting is on. This setting read
+//     is the authoritative gate; the proxy-side strip is defense in depth.
+func (m *managerImpl) saveAgentNetworkAccessLog(ctx context.Context, logEntry *accesslogs.AccessLogEntry) error {
+	entry, groups := flattenAgentNetworkLog(logEntry)
+
+	usage, usageGroups := usageFromFlattenedLog(entry, groups)
+	if err := m.store.CreateAgentNetworkUsage(ctx, usage, usageGroups); err != nil {
+		log.WithContext(ctx).WithFields(log.Fields{
+			"account_id": entry.AccountID,
+			"model":      entry.Model,
+		}).Errorf("failed to save agent-network usage: %v", err)
+		return err
+	}
+
+	settings, err := m.store.GetAgentNetworkSettings(ctx, store.LockingStrengthNone, entry.AccountID)
+	if err != nil {
+		// No settings row (or a transient read error) means we can't confirm
+		// log collection is enabled — usage is already saved, so skip the full
+		// row rather than fail the whole ingest.
+		log.WithContext(ctx).Debugf("skipping full agent-network access-log row for account %s: %v", entry.AccountID, err)
+		return nil
+	}
+	if !settings.EnableLogCollection {
+		return nil
+	}
+
+	if err := m.store.CreateAgentNetworkAccessLog(ctx, entry, groups); err != nil {
+		log.WithContext(ctx).WithFields(log.Fields{
+			"account_id": entry.AccountID,
+			"service_id": entry.ServiceID,
+			"model":      entry.Model,
+			"status":     entry.StatusCode,
+		}).Errorf("failed to save agent-network access log: %v", err)
+		return err
+	}
+	return nil
+}
+
+// flattenAgentNetworkLog converts a reverse-proxy AccessLogEntry (whose LLM
+// dimensions live in the opaque Metadata map) into the flattened
+// agent-network row + authorising-group child rows.
+func flattenAgentNetworkLog(e *accesslogs.AccessLogEntry) (*agentNetworkTypes.AgentNetworkAccessLog, []agentNetworkTypes.AgentNetworkAccessLogGroup) {
+	meta := e.Metadata
+
+	var sourceIP string
+	if e.GeoLocation.ConnectionIP != nil {
+		sourceIP = e.GeoLocation.ConnectionIP.String()
+	}
+
+	entry := &agentNetworkTypes.AgentNetworkAccessLog{
+		ID:            e.ID,
+		AccountID:     e.AccountID,
+		ServiceID:     e.ServiceID,
+		Timestamp:     e.Timestamp,
+		UserID:        e.UserId,
+		SourceIP:      sourceIP,
+		Method:        e.Method,
+		Host:          e.Host,
+		Path:          e.Path,
+		Duration:      e.Duration,
+		StatusCode:    e.StatusCode,
+		AuthMethod:    e.AuthMethodUsed,
+		BytesUpload:   e.BytesUpload,
+		BytesDownload: e.BytesDownload,
+
+		Provider:           meta[metaKeyProvider],
+		Model:              meta[metaKeyModel],
+		SessionID:          meta[metaKeySessionID],
+		ResolvedProviderID: meta[metaKeyResolvedProviderID],
+		SelectedPolicyID:   meta[metaKeySelectedPolicyID],
+		Decision:           meta[metaKeyPolicyDecision],
+		DenyReason:         meta[metaKeyPolicyReason],
+		InputTokens:        parseMetaInt(meta, metaKeyInputTokens),
+		OutputTokens:       parseMetaInt(meta, metaKeyOutputTokens),
+		TotalTokens:        parseMetaInt(meta, metaKeyTotalTokens),
+		CostUSD:            parseMetaFloat(meta, metaKeyCostUSDTotal),
+		Stream:             parseMetaBool(meta, metaKeyStream),
+		RequestPrompt:      meta[metaKeyRequestPrompt],
+		ResponseCompletion: meta[metaKeyResponseCompletion],
+	}
+
+	var groups []agentNetworkTypes.AgentNetworkAccessLogGroup
+	for _, gid := range parseGroupCSV(meta[metaKeyAuthorisingGroups]) {
+		groups = append(groups, agentNetworkTypes.AgentNetworkAccessLogGroup{
+			LogID:     entry.ID,
+			GroupID:   gid,
+			AccountID: entry.AccountID,
+		})
+	}
+	return entry, groups
+}
+
+// usageFromFlattenedLog derives the stripped usage record (and its group child
+// rows) from an already-flattened access-log entry. The usage row shares the
+// log's ID so the two correlate.
+func usageFromFlattenedLog(e *agentNetworkTypes.AgentNetworkAccessLog, groups []agentNetworkTypes.AgentNetworkAccessLogGroup) (*agentNetworkTypes.AgentNetworkUsage, []agentNetworkTypes.AgentNetworkUsageGroup) {
+	usage := &agentNetworkTypes.AgentNetworkUsage{
+		ID:                 e.ID,
+		AccountID:          e.AccountID,
+		Timestamp:          e.Timestamp,
+		UserID:             e.UserID,
+		ResolvedProviderID: e.ResolvedProviderID,
+		Provider:           e.Provider,
+		Model:              e.Model,
+		SessionID:          e.SessionID,
+		InputTokens:        e.InputTokens,
+		OutputTokens:       e.OutputTokens,
+		TotalTokens:        e.TotalTokens,
+		CostUSD:            e.CostUSD,
+	}
+
+	usageGroups := make([]agentNetworkTypes.AgentNetworkUsageGroup, 0, len(groups))
+	for _, g := range groups {
+		usageGroups = append(usageGroups, agentNetworkTypes.AgentNetworkUsageGroup{
+			UsageID:   usage.ID,
+			GroupID:   g.GroupID,
+			AccountID: g.AccountID,
+		})
+	}
+	return usage, usageGroups
+}
+
+// parseMetaInt parses a non-negative token count. Negative or unparseable
+// values are clamped to 0 so a malformed metric can't persist a negative
+// counter.
+func parseMetaInt(meta map[string]string, key string) int64 {
+	if v, err := strconv.ParseInt(strings.TrimSpace(meta[key]), 10, 64); err == nil && v >= 0 {
+		return v
+	}
+	return 0
+}
+
+// parseMetaFloat parses a non-negative, finite cost. Negative, NaN, Inf, or
+// unparseable values are clamped to 0 so a malformed metric can't poison the
+// stored cost.
+func parseMetaFloat(meta map[string]string, key string) float64 {
+	if v, err := strconv.ParseFloat(strings.TrimSpace(meta[key]), 64); err == nil && v >= 0 && !math.IsInf(v, 0) {
+		return v
+	}
+	return 0
+}
+
+func parseMetaBool(meta map[string]string, key string) bool {
+	v, _ := strconv.ParseBool(strings.TrimSpace(meta[key]))
+	return v
+}
+
+// parseGroupCSV splits the comma-separated authorising-group id list the proxy
+// emits, trimming blanks and de-duplicating. Dedup matters because the group
+// rows are keyed by (log_id, group_id) / (usage_id, group_id): a repeated id
+// in the CSV would otherwise produce a duplicate primary key and fail the
+// insert transaction.
+func parseGroupCSV(raw string) []string {
+	if raw == "" {
+		return nil
+	}
+	parts := strings.Split(raw, ",")
+	out := make([]string, 0, len(parts))
+	seen := make(map[string]struct{}, len(parts))
+	for _, p := range parts {
+		if p = strings.TrimSpace(p); p != "" {
+			if _, dup := seen[p]; dup {
+				continue
+			}
+			seen[p] = struct{}{}
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
 // GetAllAccessLogs retrieves access logs for an account with pagination and filtering
 func (m *managerImpl) GetAllAccessLogs(ctx context.Context, accountID, userID string, filter *accesslogs.AccessLogFilter) ([]*accesslogs.AccessLogEntry, int64, error) {
 	ok, ctx, err := m.permissionsManager.ValidateUserPermissions(ctx, accountID, userID, modules.Services, operations.Read)

management/internals/modules/reverseproxy/service/service.go

@@ -66,6 +66,51 @@ type TargetOptions struct {
 	// reachable without WireGuard (public APIs, LAN services, localhost
 	// sidecars). Default false.
 	DirectUpstream bool `json:"direct_upstream,omitempty"`
+	// Middlewares carries per-target agent-network middleware configs. Empty
+	// for private and operator-defined services; populated only by the
+	// agent-network synthesizer.
+	Middlewares             []MiddlewareConfig `gorm:"serializer:json" json:"middlewares,omitempty"`
+	CaptureMaxRequestBytes  int64              `json:"capture_max_request_bytes,omitempty"`
+	CaptureMaxResponseBytes int64              `json:"capture_max_response_bytes,omitempty"`
+	CaptureContentTypes     []string           `gorm:"serializer:json" json:"capture_content_types,omitempty"`
+	// AgentNetwork marks targets synthesised from Agent Network state. The
+	// proxy uses it to gate agent-network-specific behaviour (access log
+	// tagging, observability, etc.).
+	AgentNetwork bool `json:"agent_network,omitempty"`
+	// DisableAccessLog suppresses the per-request access-log emission for this
+	// target. Defaults false to preserve access-log behaviour for every
+	// non-agent-network target. The agent-network synthesizer sets this true
+	// only when the account's EnableLogCollection toggle is off.
+	DisableAccessLog bool `json:"disable_access_log,omitempty"`
+}
+
+// MiddlewareSlot mirrors proto.MiddlewareSlot / middleware.Slot.
+type MiddlewareSlot string
+
+const (
+	MiddlewareSlotOnRequest  MiddlewareSlot = "on_request"
+	MiddlewareSlotOnResponse MiddlewareSlot = "on_response"
+	MiddlewareSlotTerminal   MiddlewareSlot = "terminal"
+)
+
+// MiddlewareFailMode mirrors proto.MiddlewareConfig_FailMode.
+type MiddlewareFailMode string
+
+const (
+	MiddlewareFailOpen   MiddlewareFailMode = "fail_open"
+	MiddlewareFailClosed MiddlewareFailMode = "fail_closed"
+)
+
+// MiddlewareConfig is the per-target configuration for a single
+// middleware instance. Mirrors proto.MiddlewareConfig.
+type MiddlewareConfig struct {
+	ID         string             `json:"id"`
+	Enabled    bool               `json:"enabled"`
+	Slot       MiddlewareSlot     `json:"slot"`
+	ConfigJSON []byte             `json:"config_json,omitempty"`
+	FailMode   MiddlewareFailMode `json:"fail_mode,omitempty"`
+	TimeoutMs  int32              `json:"timeout_ms,omitempty"`
+	CanMutate  bool               `json:"can_mutate"`
 }
 
 type Target struct {
@@ -504,21 +549,75 @@ func targetOptionsToAPI(opts TargetOptions) *api.ServiceTargetOptions {
 
 func targetOptionsToProto(opts TargetOptions) *proto.PathTargetOptions {
 	if !opts.SkipTLSVerify && opts.PathRewrite == "" && opts.RequestTimeout == 0 &&
-		len(opts.CustomHeaders) == 0 && !opts.DirectUpstream {
+		len(opts.CustomHeaders) == 0 && !opts.DirectUpstream &&
+		len(opts.Middlewares) == 0 && opts.CaptureMaxRequestBytes == 0 &&
+		opts.CaptureMaxResponseBytes == 0 && len(opts.CaptureContentTypes) == 0 &&
+		!opts.AgentNetwork && !opts.DisableAccessLog {
 		return nil
 	}
 	popts := &proto.PathTargetOptions{
-		SkipTlsVerify:  opts.SkipTLSVerify,
-		PathRewrite:    pathRewriteToProto(opts.PathRewrite),
-		CustomHeaders:  opts.CustomHeaders,
-		DirectUpstream: opts.DirectUpstream,
+		SkipTlsVerify:    opts.SkipTLSVerify,
+		PathRewrite:      pathRewriteToProto(opts.PathRewrite),
+		CustomHeaders:    opts.CustomHeaders,
+		DirectUpstream:   opts.DirectUpstream,
+		AgentNetwork:     opts.AgentNetwork,
+		DisableAccessLog: opts.DisableAccessLog,
 	}
 	if opts.RequestTimeout != 0 {
 		popts.RequestTimeout = durationpb.New(opts.RequestTimeout)
 	}
+	if len(opts.Middlewares) > 0 {
+		popts.Middlewares = middlewaresToProto(opts.Middlewares)
+	}
+	popts.CaptureMaxRequestBytes = opts.CaptureMaxRequestBytes
+	popts.CaptureMaxResponseBytes = opts.CaptureMaxResponseBytes
+	if len(opts.CaptureContentTypes) > 0 {
+		popts.CaptureContentTypes = append([]string(nil), opts.CaptureContentTypes...)
+	}
 	return popts
 }
 
+// middlewaresToProto converts the internal middleware slice to the proto
+// representation sent to the proxy via the mapping stream.
+func middlewaresToProto(in []MiddlewareConfig) []*proto.MiddlewareConfig {
+	out := make([]*proto.MiddlewareConfig, 0, len(in))
+	for _, m := range in {
+		pm := &proto.MiddlewareConfig{
+			Id:         m.ID,
+			Enabled:    m.Enabled,
+			Slot:       middlewareSlotToProto(m.Slot),
+			ConfigJson: append([]byte(nil), m.ConfigJSON...),
+			CanMutate:  m.CanMutate,
+			FailMode:   middlewareFailModeToProto(m.FailMode),
+		}
+		if m.TimeoutMs > 0 {
+			pm.Timeout = durationpb.New(time.Duration(m.TimeoutMs) * time.Millisecond)
+		}
+		out = append(out, pm)
+	}
+	return out
+}
+
+func middlewareSlotToProto(s MiddlewareSlot) proto.MiddlewareSlot {
+	switch s {
+	case MiddlewareSlotOnRequest:
+		return proto.MiddlewareSlot_MIDDLEWARE_SLOT_ON_REQUEST
+	case MiddlewareSlotOnResponse:
+		return proto.MiddlewareSlot_MIDDLEWARE_SLOT_ON_RESPONSE
+	case MiddlewareSlotTerminal:
+		return proto.MiddlewareSlot_MIDDLEWARE_SLOT_TERMINAL
+	default:
+		return proto.MiddlewareSlot_MIDDLEWARE_SLOT_UNSPECIFIED
+	}
+}
+
+func middlewareFailModeToProto(m MiddlewareFailMode) proto.MiddlewareConfig_FailMode {
+	if m == MiddlewareFailClosed {
+		return proto.MiddlewareConfig_FAIL_CLOSED
+	}
+	return proto.MiddlewareConfig_FAIL_OPEN
+}
+
 // l4TargetOptionsToProto converts L4-relevant target options to proto.
 func l4TargetOptionsToProto(target *Target) *proto.PathTargetOptions {
 	if !target.ProxyProtocol && target.Options.RequestTimeout == 0 && target.Options.SessionIdleTimeout == 0 {