mirror of
https://github.com/netbirdio/netbird.git
synced 2026-09-19 05:09:06 +02:00
Add session view support in the access log
This commit is contained in:
@@ -33,11 +33,11 @@ import (
|
||||
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/domain"
|
||||
|
||||
agentNetworkTypes "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy"
|
||||
rpservice "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/zones"
|
||||
"github.com/netbirdio/netbird/management/internals/modules/zones/records"
|
||||
agentNetworkTypes "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
|
||||
resourceTypes "github.com/netbirdio/netbird/management/server/networks/resources/types"
|
||||
routerTypes "github.com/netbirdio/netbird/management/server/networks/routers/types"
|
||||
networkTypes "github.com/netbirdio/netbird/management/server/networks/types"
|
||||
@@ -5827,6 +5827,91 @@ func (s *SqlStore) hydrateAgentNetworkAccessLogGroups(ctx context.Context, accou
|
||||
return nil
|
||||
}
|
||||
|
||||
// agentNetworkSessionKeyExpr is the SQL group key for session-grouped access
|
||||
// logs: the row's session id, or — when the client sent none — the row id, so
|
||||
// session-less requests each form their own singleton group. COALESCE/NULLIF
|
||||
// are standard SQL, so this stays portable across SQLite and Postgres.
|
||||
const agentNetworkSessionKeyExpr = "COALESCE(NULLIF(session_id, ''), id)"
|
||||
|
||||
// GetAgentNetworkAccessLogSessions retrieves agent-network access logs grouped
|
||||
// by session, with server-side pagination, filtering and sorting at the session
|
||||
// level. It paginates over the distinct session keys (ordered by the requested
|
||||
// session-level aggregate), fetches every entry for the page's sessions, and
|
||||
// folds them into per-session summaries. The returned count is the number of
|
||||
// matching sessions. Filters apply to the entries, so a session's summary
|
||||
// reflects only its filter-matching requests.
|
||||
func (s *SqlStore) GetAgentNetworkAccessLogSessions(ctx context.Context, lockStrength LockingStrength, accountID string, filter agentNetworkTypes.AgentNetworkAccessLogFilter) ([]*agentNetworkTypes.AgentNetworkAccessLogSession, int64, error) {
|
||||
// Count distinct sessions via a grouped subquery — portable and avoids
|
||||
// relying on COUNT(DISTINCT <expr>) quoting quirks.
|
||||
sessionsSubquery := s.applyAgentNetworkAccessLogFilters(
|
||||
s.db.Model(&agentNetworkTypes.AgentNetworkAccessLog{}).Where(accountIDCondition, accountID),
|
||||
filter,
|
||||
).
|
||||
Select(agentNetworkSessionKeyExpr + " AS session_key").
|
||||
Group(agentNetworkSessionKeyExpr)
|
||||
|
||||
var totalCount int64
|
||||
if err := s.db.Table("(?) AS sessions", sessionsSubquery).Count(&totalCount).Error; err != nil {
|
||||
log.WithContext(ctx).Errorf("failed to count agent-network access-log sessions: %v", err)
|
||||
return nil, 0, status.Errorf(status.Internal, "failed to count agent-network access-log sessions")
|
||||
}
|
||||
|
||||
// The page of session keys, ordered by the session-level aggregate. The
|
||||
// session-key tiebreaker keeps pagination deterministic when the primary
|
||||
// aggregate ties.
|
||||
type sessionKeyRow struct {
|
||||
SessionKey string
|
||||
}
|
||||
var keyRows []sessionKeyRow
|
||||
keyQuery := s.applyAgentNetworkAccessLogFilters(
|
||||
s.db.Model(&agentNetworkTypes.AgentNetworkAccessLog{}).Where(accountIDCondition, accountID),
|
||||
filter,
|
||||
).
|
||||
Select(agentNetworkSessionKeyExpr + " AS session_key").
|
||||
Group(agentNetworkSessionKeyExpr).
|
||||
Order(filter.GetSessionSortExpr() + " " + filter.GetSortOrder()).
|
||||
Order("session_key ASC").
|
||||
Limit(filter.GetLimit()).
|
||||
Offset(filter.GetOffset())
|
||||
if err := keyQuery.Scan(&keyRows).Error; err != nil {
|
||||
log.WithContext(ctx).Errorf("failed to list agent-network access-log session keys: %v", err)
|
||||
return nil, 0, status.Errorf(status.Internal, "failed to list agent-network access-log session keys")
|
||||
}
|
||||
if len(keyRows) == 0 {
|
||||
return nil, totalCount, nil
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(keyRows))
|
||||
for _, r := range keyRows {
|
||||
keys = append(keys, r.SessionKey)
|
||||
}
|
||||
|
||||
// All entries for the page's sessions, contiguous per session and oldest
|
||||
// first within each — the fold relies on that ordering.
|
||||
var entries []*agentNetworkTypes.AgentNetworkAccessLog
|
||||
entriesQuery := s.applyAgentNetworkAccessLogFilters(
|
||||
s.db.Where(accountIDCondition, accountID),
|
||||
filter,
|
||||
).
|
||||
Where(agentNetworkSessionKeyExpr+" IN ?", keys).
|
||||
Order(agentNetworkSessionKeyExpr + ", timestamp ASC")
|
||||
|
||||
if lockStrength != LockingStrengthNone {
|
||||
entriesQuery = entriesQuery.Clauses(clause.Locking{Strength: string(lockStrength)})
|
||||
}
|
||||
|
||||
if err := entriesQuery.Find(&entries).Error; err != nil {
|
||||
log.WithContext(ctx).Errorf("failed to get agent-network access-log session entries: %v", err)
|
||||
return nil, 0, status.Errorf(status.Internal, "failed to get agent-network access-log session entries")
|
||||
}
|
||||
|
||||
if err := s.hydrateAgentNetworkAccessLogGroups(ctx, accountID, entries); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
return agentNetworkTypes.FoldAccessLogSessions(keys, entries), totalCount, nil
|
||||
}
|
||||
|
||||
// GetAccountAccessLogs retrieves access logs for a given account with pagination and filtering
|
||||
func (s *SqlStore) GetAccountAccessLogs(ctx context.Context, lockStrength LockingStrength, accountID string, filter accesslogs.AccessLogFilter) ([]*accesslogs.AccessLogEntry, int64, error) {
|
||||
var logs []*accesslogs.AccessLogEntry
|
||||
|
||||
@@ -155,6 +155,104 @@ func TestAgentNetworkUsageOverview_DailyAggregation(t *testing.T) {
|
||||
assert.Equal(t, "u3", filtered[0].ID)
|
||||
}
|
||||
|
||||
// TestAgentNetworkAccessLogSessions_RealStore drives GetAgentNetworkAccessLogSessions
|
||||
// against a real sqlite store: session grouping + aggregation, recency ordering,
|
||||
// singleton groups for session-less requests, session pagination, the model
|
||||
// filter narrowing sessions, and aggregate sorting.
|
||||
func TestAgentNetworkAccessLogSessions_RealStore(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, cleanup, err := NewTestStoreFromSQL(ctx, "", t.TempDir())
|
||||
require.NoError(t, err, "real sqlite test store must come up")
|
||||
defer cleanup()
|
||||
|
||||
const accountID = "acc-anet-sessions-1"
|
||||
base := time.Date(2026, 5, 5, 10, 0, 0, 0, time.UTC)
|
||||
at := func(h int) time.Time { return base.Add(time.Duration(h) * time.Hour) }
|
||||
|
||||
mk := func(id, session, user, provider, model, decision string, ts time.Time, cost float64) *agentNetworkTypes.AgentNetworkAccessLog {
|
||||
return &agentNetworkTypes.AgentNetworkAccessLog{
|
||||
ID: id, AccountID: accountID, ServiceID: "svc", Timestamp: ts,
|
||||
UserID: user, StatusCode: 200, Provider: provider, Model: model,
|
||||
SessionID: session, Decision: decision,
|
||||
InputTokens: 100, OutputTokens: 50, TotalTokens: 150, CostUSD: cost,
|
||||
}
|
||||
}
|
||||
|
||||
// Two-request session s1 (alice), a one-request denied session s2 (bob), and
|
||||
// two session-less requests (empty session id) that must each form their own
|
||||
// singleton group.
|
||||
require.NoError(t, s.CreateAgentNetworkAccessLog(ctx, mk("s1-a", "s1", "alice", "openai", "gpt-4o", "allow", at(1), 0.10),
|
||||
[]agentNetworkTypes.AgentNetworkAccessLogGroup{{LogID: "s1-a", GroupID: "grp-eng", AccountID: accountID}}))
|
||||
require.NoError(t, s.CreateAgentNetworkAccessLog(ctx, mk("s1-b", "s1", "alice", "openai", "gpt-4o", "allow", at(2), 0.20),
|
||||
[]agentNetworkTypes.AgentNetworkAccessLogGroup{{LogID: "s1-b", GroupID: "grp-oncall", AccountID: accountID}}))
|
||||
require.NoError(t, s.CreateAgentNetworkAccessLog(ctx, mk("s2-a", "s2", "bob", "anthropic", "claude-3", "deny", at(3), 0.05), nil))
|
||||
require.NoError(t, s.CreateAgentNetworkAccessLog(ctx, mk("se-old", "", "carol", "openai", "o1", "allow", at(0), 0.01), nil))
|
||||
require.NoError(t, s.CreateAgentNetworkAccessLog(ctx, mk("se-new", "", "dave", "mistral", "mistral-large", "allow", at(4), 0.02), nil))
|
||||
|
||||
// Default sort: last activity (MAX timestamp) descending.
|
||||
sessions, total, err := s.GetAgentNetworkAccessLogSessions(ctx, LockingStrengthNone, accountID,
|
||||
agentNetworkTypes.AgentNetworkAccessLogFilter{Page: 1, PageSize: 50})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(4), total, "four sessions: s1, s2, and two singletons")
|
||||
require.Len(t, sessions, 4)
|
||||
|
||||
// se-new(t4) > s2(t3) > s1(t2) > se-old(t0)
|
||||
assert.Equal(t, "", sessions[0].SessionID, "newest is a session-less singleton")
|
||||
assert.Equal(t, "se-new", sessions[0].Entries[0].ID)
|
||||
assert.Equal(t, "s2", sessions[1].SessionID)
|
||||
assert.Equal(t, "s1", sessions[2].SessionID)
|
||||
assert.Equal(t, "se-old", sessions[3].Entries[0].ID)
|
||||
|
||||
// s1 aggregation.
|
||||
s1 := sessions[2]
|
||||
assert.Equal(t, 2, s1.RequestCount, "s1 has two requests")
|
||||
assert.Equal(t, int64(300), s1.TotalTokens, "tokens summed across the session")
|
||||
assert.InDelta(t, 0.30, s1.CostUSD, 1e-9, "cost summed across the session")
|
||||
assert.Equal(t, "alice", s1.UserID)
|
||||
assert.Equal(t, "allow", s1.Decision)
|
||||
assert.Equal(t, at(1), s1.StartedAt, "started = earliest entry")
|
||||
assert.Equal(t, at(2), s1.EndedAt, "ended = latest entry")
|
||||
assert.ElementsMatch(t, []string{"openai"}, s1.Providers)
|
||||
assert.ElementsMatch(t, []string{"gpt-4o"}, s1.Models)
|
||||
assert.ElementsMatch(t, []string{"grp-eng", "grp-oncall"}, s1.GroupIDs, "union of the entries' authorising groups")
|
||||
|
||||
// Denied session rolls up to deny.
|
||||
assert.Equal(t, "deny", sessions[1].Decision, "any denied request makes the session deny")
|
||||
|
||||
// Pagination over sessions: 2 per page.
|
||||
page1, total, err := s.GetAgentNetworkAccessLogSessions(ctx, LockingStrengthNone, accountID,
|
||||
agentNetworkTypes.AgentNetworkAccessLogFilter{Page: 1, PageSize: 2})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(4), total, "total still counts all sessions")
|
||||
require.Len(t, page1, 2)
|
||||
assert.Equal(t, "se-new", page1[0].Entries[0].ID)
|
||||
assert.Equal(t, "s2", page1[1].SessionID)
|
||||
|
||||
page2, _, err := s.GetAgentNetworkAccessLogSessions(ctx, LockingStrengthNone, accountID,
|
||||
agentNetworkTypes.AgentNetworkAccessLogFilter{Page: 2, PageSize: 2})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, page2, 2)
|
||||
assert.Equal(t, "s1", page2[0].SessionID)
|
||||
assert.Equal(t, "se-old", page2[1].Entries[0].ID)
|
||||
|
||||
// Model filter narrows to the session(s) with matching entries.
|
||||
model := "claude-3"
|
||||
filtered, fTotal, err := s.GetAgentNetworkAccessLogSessions(ctx, LockingStrengthNone, accountID,
|
||||
agentNetworkTypes.AgentNetworkAccessLogFilter{Page: 1, PageSize: 50, Models: []string{model}})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(1), fTotal, "only s2 has a claude-3 request")
|
||||
require.Len(t, filtered, 1)
|
||||
assert.Equal(t, "s2", filtered[0].SessionID)
|
||||
|
||||
// Sort by total session cost, descending: s1 (0.30) leads despite not being
|
||||
// the most recent.
|
||||
byCost, _, err := s.GetAgentNetworkAccessLogSessions(ctx, LockingStrengthNone, accountID,
|
||||
agentNetworkTypes.AgentNetworkAccessLogFilter{Page: 1, PageSize: 50, SortBy: "cost_usd", SortOrder: "desc"})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, byCost, 4)
|
||||
assert.Equal(t, "s1", byCost[0].SessionID, "highest-cost session sorts first")
|
||||
}
|
||||
|
||||
// TestDeleteOldAgentNetworkAccessLogs verifies the retention sweep removes only
|
||||
// access-log rows (and their group children) older than the cutoff, leaving
|
||||
// recent rows — and never touching usage records.
|
||||
|
||||
@@ -304,6 +304,7 @@ type Store interface {
|
||||
CreateAgentNetworkAccessLog(ctx context.Context, entry *agentNetworkTypes.AgentNetworkAccessLog, groups []agentNetworkTypes.AgentNetworkAccessLogGroup) error
|
||||
CreateAgentNetworkUsage(ctx context.Context, usage *agentNetworkTypes.AgentNetworkUsage, groups []agentNetworkTypes.AgentNetworkUsageGroup) error
|
||||
GetAgentNetworkAccessLogs(ctx context.Context, lockStrength LockingStrength, accountID string, filter agentNetworkTypes.AgentNetworkAccessLogFilter) ([]*agentNetworkTypes.AgentNetworkAccessLog, int64, error)
|
||||
GetAgentNetworkAccessLogSessions(ctx context.Context, lockStrength LockingStrength, accountID string, filter agentNetworkTypes.AgentNetworkAccessLogFilter) ([]*agentNetworkTypes.AgentNetworkAccessLogSession, int64, error)
|
||||
GetAgentNetworkUsageRows(ctx context.Context, lockStrength LockingStrength, accountID string, filter agentNetworkTypes.AgentNetworkAccessLogFilter) ([]*agentNetworkTypes.AgentNetworkUsage, error)
|
||||
DeleteOldAgentNetworkAccessLogs(ctx context.Context, accountID string, olderThan time.Time) (int64, error)
|
||||
GetServiceTargetByTargetID(ctx context.Context, lockStrength LockingStrength, accountID string, targetID string) (*rpservice.Target, error)
|
||||
|
||||
@@ -433,6 +433,22 @@ func (mr *MockStoreMockRecorder) GetAgentNetworkAccessLogs(ctx, lockStrength, ac
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAgentNetworkAccessLogs", reflect.TypeOf((*MockStore)(nil).GetAgentNetworkAccessLogs), ctx, lockStrength, accountID, filter)
|
||||
}
|
||||
|
||||
// GetAgentNetworkAccessLogSessions mocks base method.
|
||||
func (m *MockStore) GetAgentNetworkAccessLogSessions(ctx context.Context, lockStrength LockingStrength, accountID string, filter agentNetworkTypes.AgentNetworkAccessLogFilter) ([]*agentNetworkTypes.AgentNetworkAccessLogSession, int64, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "GetAgentNetworkAccessLogSessions", ctx, lockStrength, accountID, filter)
|
||||
ret0, _ := ret[0].([]*agentNetworkTypes.AgentNetworkAccessLogSession)
|
||||
ret1, _ := ret[1].(int64)
|
||||
ret2, _ := ret[2].(error)
|
||||
return ret0, ret1, ret2
|
||||
}
|
||||
|
||||
// GetAgentNetworkAccessLogSessions indicates an expected call of GetAgentNetworkAccessLogSessions.
|
||||
func (mr *MockStoreMockRecorder) GetAgentNetworkAccessLogSessions(ctx, lockStrength, accountID, filter interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAgentNetworkAccessLogSessions", reflect.TypeOf((*MockStore)(nil).GetAgentNetworkAccessLogSessions), ctx, lockStrength, accountID, filter)
|
||||
}
|
||||
|
||||
// GetAgentNetworkUsageRows mocks base method.
|
||||
func (m *MockStore) GetAgentNetworkUsageRows(ctx context.Context, lockStrength LockingStrength, accountID string, filter agentNetworkTypes.AgentNetworkAccessLogFilter) ([]*agentNetworkTypes.AgentNetworkUsage, error) {
|
||||
m.ctrl.T.Helper()
|
||||
|
||||
Reference in New Issue
Block a user