Add session view support in the access log

This commit is contained in:
braginini
2026-06-30 19:04:04 +02:00
parent 8f9d35ef73
commit 041a8fe252
10 changed files with 856 additions and 9 deletions
+86 -1
View File
@@ -33,11 +33,11 @@ import (
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/accesslogs"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/domain"
agentNetworkTypes "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy"
rpservice "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service"
"github.com/netbirdio/netbird/management/internals/modules/zones"
"github.com/netbirdio/netbird/management/internals/modules/zones/records"
agentNetworkTypes "github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
resourceTypes "github.com/netbirdio/netbird/management/server/networks/resources/types"
routerTypes "github.com/netbirdio/netbird/management/server/networks/routers/types"
networkTypes "github.com/netbirdio/netbird/management/server/networks/types"
@@ -5827,6 +5827,91 @@ func (s *SqlStore) hydrateAgentNetworkAccessLogGroups(ctx context.Context, accou
return nil
}
// agentNetworkSessionKeyExpr is the SQL group key for session-grouped access
// logs: the row's session id, or — when the client sent none — the row id, so
// session-less requests each form their own singleton group. COALESCE/NULLIF
// are standard SQL, so this stays portable across SQLite and Postgres.
const agentNetworkSessionKeyExpr = "COALESCE(NULLIF(session_id, ''), id)"
// GetAgentNetworkAccessLogSessions retrieves agent-network access logs grouped
// by session, with server-side pagination, filtering and sorting at the session
// level. It paginates over the distinct session keys (ordered by the requested
// session-level aggregate), fetches every entry for the page's sessions, and
// folds them into per-session summaries. The returned count is the number of
// matching sessions. Filters apply to the entries, so a session's summary
// reflects only its filter-matching requests.
func (s *SqlStore) GetAgentNetworkAccessLogSessions(ctx context.Context, lockStrength LockingStrength, accountID string, filter agentNetworkTypes.AgentNetworkAccessLogFilter) ([]*agentNetworkTypes.AgentNetworkAccessLogSession, int64, error) {
// Count distinct sessions via a grouped subquery — portable and avoids
// relying on COUNT(DISTINCT <expr>) quoting quirks.
sessionsSubquery := s.applyAgentNetworkAccessLogFilters(
s.db.Model(&agentNetworkTypes.AgentNetworkAccessLog{}).Where(accountIDCondition, accountID),
filter,
).
Select(agentNetworkSessionKeyExpr + " AS session_key").
Group(agentNetworkSessionKeyExpr)
var totalCount int64
if err := s.db.Table("(?) AS sessions", sessionsSubquery).Count(&totalCount).Error; err != nil {
log.WithContext(ctx).Errorf("failed to count agent-network access-log sessions: %v", err)
return nil, 0, status.Errorf(status.Internal, "failed to count agent-network access-log sessions")
}
// The page of session keys, ordered by the session-level aggregate. The
// session-key tiebreaker keeps pagination deterministic when the primary
// aggregate ties.
type sessionKeyRow struct {
SessionKey string
}
var keyRows []sessionKeyRow
keyQuery := s.applyAgentNetworkAccessLogFilters(
s.db.Model(&agentNetworkTypes.AgentNetworkAccessLog{}).Where(accountIDCondition, accountID),
filter,
).
Select(agentNetworkSessionKeyExpr + " AS session_key").
Group(agentNetworkSessionKeyExpr).
Order(filter.GetSessionSortExpr() + " " + filter.GetSortOrder()).
Order("session_key ASC").
Limit(filter.GetLimit()).
Offset(filter.GetOffset())
if err := keyQuery.Scan(&keyRows).Error; err != nil {
log.WithContext(ctx).Errorf("failed to list agent-network access-log session keys: %v", err)
return nil, 0, status.Errorf(status.Internal, "failed to list agent-network access-log session keys")
}
if len(keyRows) == 0 {
return nil, totalCount, nil
}
keys := make([]string, 0, len(keyRows))
for _, r := range keyRows {
keys = append(keys, r.SessionKey)
}
// All entries for the page's sessions, contiguous per session and oldest
// first within each — the fold relies on that ordering.
var entries []*agentNetworkTypes.AgentNetworkAccessLog
entriesQuery := s.applyAgentNetworkAccessLogFilters(
s.db.Where(accountIDCondition, accountID),
filter,
).
Where(agentNetworkSessionKeyExpr+" IN ?", keys).
Order(agentNetworkSessionKeyExpr + ", timestamp ASC")
if lockStrength != LockingStrengthNone {
entriesQuery = entriesQuery.Clauses(clause.Locking{Strength: string(lockStrength)})
}
if err := entriesQuery.Find(&entries).Error; err != nil {
log.WithContext(ctx).Errorf("failed to get agent-network access-log session entries: %v", err)
return nil, 0, status.Errorf(status.Internal, "failed to get agent-network access-log session entries")
}
if err := s.hydrateAgentNetworkAccessLogGroups(ctx, accountID, entries); err != nil {
return nil, 0, err
}
return agentNetworkTypes.FoldAccessLogSessions(keys, entries), totalCount, nil
}
// GetAccountAccessLogs retrieves access logs for a given account with pagination and filtering
func (s *SqlStore) GetAccountAccessLogs(ctx context.Context, lockStrength LockingStrength, accountID string, filter accesslogs.AccessLogFilter) ([]*accesslogs.AccessLogEntry, int64, error) {
var logs []*accesslogs.AccessLogEntry
@@ -155,6 +155,104 @@ func TestAgentNetworkUsageOverview_DailyAggregation(t *testing.T) {
assert.Equal(t, "u3", filtered[0].ID)
}
// TestAgentNetworkAccessLogSessions_RealStore drives GetAgentNetworkAccessLogSessions
// against a real sqlite store: session grouping + aggregation, recency ordering,
// singleton groups for session-less requests, session pagination, the model
// filter narrowing sessions, and aggregate sorting.
func TestAgentNetworkAccessLogSessions_RealStore(t *testing.T) {
ctx := context.Background()
s, cleanup, err := NewTestStoreFromSQL(ctx, "", t.TempDir())
require.NoError(t, err, "real sqlite test store must come up")
defer cleanup()
const accountID = "acc-anet-sessions-1"
base := time.Date(2026, 5, 5, 10, 0, 0, 0, time.UTC)
at := func(h int) time.Time { return base.Add(time.Duration(h) * time.Hour) }
mk := func(id, session, user, provider, model, decision string, ts time.Time, cost float64) *agentNetworkTypes.AgentNetworkAccessLog {
return &agentNetworkTypes.AgentNetworkAccessLog{
ID: id, AccountID: accountID, ServiceID: "svc", Timestamp: ts,
UserID: user, StatusCode: 200, Provider: provider, Model: model,
SessionID: session, Decision: decision,
InputTokens: 100, OutputTokens: 50, TotalTokens: 150, CostUSD: cost,
}
}
// Two-request session s1 (alice), a one-request denied session s2 (bob), and
// two session-less requests (empty session id) that must each form their own
// singleton group.
require.NoError(t, s.CreateAgentNetworkAccessLog(ctx, mk("s1-a", "s1", "alice", "openai", "gpt-4o", "allow", at(1), 0.10),
[]agentNetworkTypes.AgentNetworkAccessLogGroup{{LogID: "s1-a", GroupID: "grp-eng", AccountID: accountID}}))
require.NoError(t, s.CreateAgentNetworkAccessLog(ctx, mk("s1-b", "s1", "alice", "openai", "gpt-4o", "allow", at(2), 0.20),
[]agentNetworkTypes.AgentNetworkAccessLogGroup{{LogID: "s1-b", GroupID: "grp-oncall", AccountID: accountID}}))
require.NoError(t, s.CreateAgentNetworkAccessLog(ctx, mk("s2-a", "s2", "bob", "anthropic", "claude-3", "deny", at(3), 0.05), nil))
require.NoError(t, s.CreateAgentNetworkAccessLog(ctx, mk("se-old", "", "carol", "openai", "o1", "allow", at(0), 0.01), nil))
require.NoError(t, s.CreateAgentNetworkAccessLog(ctx, mk("se-new", "", "dave", "mistral", "mistral-large", "allow", at(4), 0.02), nil))
// Default sort: last activity (MAX timestamp) descending.
sessions, total, err := s.GetAgentNetworkAccessLogSessions(ctx, LockingStrengthNone, accountID,
agentNetworkTypes.AgentNetworkAccessLogFilter{Page: 1, PageSize: 50})
require.NoError(t, err)
assert.Equal(t, int64(4), total, "four sessions: s1, s2, and two singletons")
require.Len(t, sessions, 4)
// se-new(t4) > s2(t3) > s1(t2) > se-old(t0)
assert.Equal(t, "", sessions[0].SessionID, "newest is a session-less singleton")
assert.Equal(t, "se-new", sessions[0].Entries[0].ID)
assert.Equal(t, "s2", sessions[1].SessionID)
assert.Equal(t, "s1", sessions[2].SessionID)
assert.Equal(t, "se-old", sessions[3].Entries[0].ID)
// s1 aggregation.
s1 := sessions[2]
assert.Equal(t, 2, s1.RequestCount, "s1 has two requests")
assert.Equal(t, int64(300), s1.TotalTokens, "tokens summed across the session")
assert.InDelta(t, 0.30, s1.CostUSD, 1e-9, "cost summed across the session")
assert.Equal(t, "alice", s1.UserID)
assert.Equal(t, "allow", s1.Decision)
assert.Equal(t, at(1), s1.StartedAt, "started = earliest entry")
assert.Equal(t, at(2), s1.EndedAt, "ended = latest entry")
assert.ElementsMatch(t, []string{"openai"}, s1.Providers)
assert.ElementsMatch(t, []string{"gpt-4o"}, s1.Models)
assert.ElementsMatch(t, []string{"grp-eng", "grp-oncall"}, s1.GroupIDs, "union of the entries' authorising groups")
// Denied session rolls up to deny.
assert.Equal(t, "deny", sessions[1].Decision, "any denied request makes the session deny")
// Pagination over sessions: 2 per page.
page1, total, err := s.GetAgentNetworkAccessLogSessions(ctx, LockingStrengthNone, accountID,
agentNetworkTypes.AgentNetworkAccessLogFilter{Page: 1, PageSize: 2})
require.NoError(t, err)
assert.Equal(t, int64(4), total, "total still counts all sessions")
require.Len(t, page1, 2)
assert.Equal(t, "se-new", page1[0].Entries[0].ID)
assert.Equal(t, "s2", page1[1].SessionID)
page2, _, err := s.GetAgentNetworkAccessLogSessions(ctx, LockingStrengthNone, accountID,
agentNetworkTypes.AgentNetworkAccessLogFilter{Page: 2, PageSize: 2})
require.NoError(t, err)
require.Len(t, page2, 2)
assert.Equal(t, "s1", page2[0].SessionID)
assert.Equal(t, "se-old", page2[1].Entries[0].ID)
// Model filter narrows to the session(s) with matching entries.
model := "claude-3"
filtered, fTotal, err := s.GetAgentNetworkAccessLogSessions(ctx, LockingStrengthNone, accountID,
agentNetworkTypes.AgentNetworkAccessLogFilter{Page: 1, PageSize: 50, Models: []string{model}})
require.NoError(t, err)
assert.Equal(t, int64(1), fTotal, "only s2 has a claude-3 request")
require.Len(t, filtered, 1)
assert.Equal(t, "s2", filtered[0].SessionID)
// Sort by total session cost, descending: s1 (0.30) leads despite not being
// the most recent.
byCost, _, err := s.GetAgentNetworkAccessLogSessions(ctx, LockingStrengthNone, accountID,
agentNetworkTypes.AgentNetworkAccessLogFilter{Page: 1, PageSize: 50, SortBy: "cost_usd", SortOrder: "desc"})
require.NoError(t, err)
require.Len(t, byCost, 4)
assert.Equal(t, "s1", byCost[0].SessionID, "highest-cost session sorts first")
}
// TestDeleteOldAgentNetworkAccessLogs verifies the retention sweep removes only
// access-log rows (and their group children) older than the cutoff, leaving
// recent rows — and never touching usage records.
+1
View File
@@ -304,6 +304,7 @@ type Store interface {
CreateAgentNetworkAccessLog(ctx context.Context, entry *agentNetworkTypes.AgentNetworkAccessLog, groups []agentNetworkTypes.AgentNetworkAccessLogGroup) error
CreateAgentNetworkUsage(ctx context.Context, usage *agentNetworkTypes.AgentNetworkUsage, groups []agentNetworkTypes.AgentNetworkUsageGroup) error
GetAgentNetworkAccessLogs(ctx context.Context, lockStrength LockingStrength, accountID string, filter agentNetworkTypes.AgentNetworkAccessLogFilter) ([]*agentNetworkTypes.AgentNetworkAccessLog, int64, error)
GetAgentNetworkAccessLogSessions(ctx context.Context, lockStrength LockingStrength, accountID string, filter agentNetworkTypes.AgentNetworkAccessLogFilter) ([]*agentNetworkTypes.AgentNetworkAccessLogSession, int64, error)
GetAgentNetworkUsageRows(ctx context.Context, lockStrength LockingStrength, accountID string, filter agentNetworkTypes.AgentNetworkAccessLogFilter) ([]*agentNetworkTypes.AgentNetworkUsage, error)
DeleteOldAgentNetworkAccessLogs(ctx context.Context, accountID string, olderThan time.Time) (int64, error)
GetServiceTargetByTargetID(ctx context.Context, lockStrength LockingStrength, accountID string, targetID string) (*rpservice.Target, error)
@@ -433,6 +433,22 @@ func (mr *MockStoreMockRecorder) GetAgentNetworkAccessLogs(ctx, lockStrength, ac
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAgentNetworkAccessLogs", reflect.TypeOf((*MockStore)(nil).GetAgentNetworkAccessLogs), ctx, lockStrength, accountID, filter)
}
// GetAgentNetworkAccessLogSessions mocks base method.
func (m *MockStore) GetAgentNetworkAccessLogSessions(ctx context.Context, lockStrength LockingStrength, accountID string, filter agentNetworkTypes.AgentNetworkAccessLogFilter) ([]*agentNetworkTypes.AgentNetworkAccessLogSession, int64, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetAgentNetworkAccessLogSessions", ctx, lockStrength, accountID, filter)
ret0, _ := ret[0].([]*agentNetworkTypes.AgentNetworkAccessLogSession)
ret1, _ := ret[1].(int64)
ret2, _ := ret[2].(error)
return ret0, ret1, ret2
}
// GetAgentNetworkAccessLogSessions indicates an expected call of GetAgentNetworkAccessLogSessions.
func (mr *MockStoreMockRecorder) GetAgentNetworkAccessLogSessions(ctx, lockStrength, accountID, filter interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAgentNetworkAccessLogSessions", reflect.TypeOf((*MockStore)(nil).GetAgentNetworkAccessLogSessions), ctx, lockStrength, accountID, filter)
}
// GetAgentNetworkUsageRows mocks base method.
func (m *MockStore) GetAgentNetworkUsageRows(ctx context.Context, lockStrength LockingStrength, accountID string, filter agentNetworkTypes.AgentNetworkAccessLogFilter) ([]*agentNetworkTypes.AgentNetworkUsage, error) {
m.ctrl.T.Helper()