[management] Clean up after account deletion (#7812)

Deleting an account left state behind that DeleteAccount's store
associations don't reach. The Agent Network tables outlived the account,
keeping its gateway domain claimed and its provider API keys stored. The
proxies kept serving its gateway until they next resynced. Cloud-side
state, such as managed proxy deployments, had no way to be cleaned up at
all.

Account deletion now runs registered hooks after the permission check
and before any users or data are removed. A failing hook aborts the
deletion. Agent Network registers one that tells the proxies to drop the
account's gateway mappings. The account's settings, providers, policies,
guardrails and budget rules are deleted in the account's transaction.
Consumption counters, and the access logs of deleted accounts, are left
to the background cleanup; usage records are kept.

management/internals/modules/agentnetwork/accesslog_cleanup_realstore_test.go

@@ -0,0 +1,76 @@
+package agentnetwork
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/netbirdio/netbird/management/internals/modules/agentnetwork/types"
+	"github.com/netbirdio/netbird/management/server/store"
+	nbtypes "github.com/netbirdio/netbird/management/server/types"
+)
+
+// TestCleanupAccessLogs_RealStore_DeletedAccount covers a deleted account's access logs.
+// The sweep is driven by settings rows, which go with the account, so without a fallback
+// those logs would never expire. They get the default retention instead. A live account
+// can delete its own settings row, so "no settings" must not be mistaken for "deleted":
+// that account's logs are left alone, as are those of an account that keeps logs forever.
+func TestCleanupAccessLogs_RealStore_DeletedAccount(t *testing.T) {
+	ctx := context.Background()
+	s, cleanup, err := store.NewTestStoreFromSQL(ctx, "", t.TempDir())
+	require.NoError(t, err, "real sqlite test store must come up")
+	defer cleanup()
+
+	const (
+		deletedAccountID    = "acc-deleted"
+		keepAccountID       = "acc-keep-forever"
+		noSettingsAccountID = "acc-live-no-settings"
+	)
+	old := time.Now().UTC().AddDate(0, 0, -(types.DefaultAccessLogRetentionDays + 10))
+	recent := time.Now().UTC().AddDate(0, 0, -1)
+
+	require.NoError(t, s.SaveAccount(ctx, &nbtypes.Account{Id: keepAccountID}))
+	require.NoError(t, s.SaveAccount(ctx, &nbtypes.Account{Id: noSettingsAccountID}))
+
+	keepSettings := types.DefaultSettings(keepAccountID)
+	keepSettings.Domain = "keep.gw.example.com"
+	keepSettings.AccessLogRetentionDays = 0
+	require.NoError(t, s.SaveAgentNetworkSettings(ctx, keepSettings))
+
+	mkLog := func(id, accountID string, ts time.Time) {
+		t.Helper()
+		entry := &types.AgentNetworkAccessLog{
+			ID: id, AccountID: accountID, ServiceID: "svc", Timestamp: ts, StatusCode: 200, Model: "gpt-4o",
+		}
+		groups := []types.AgentNetworkAccessLogGroup{{LogID: id, GroupID: "grp-eng", AccountID: accountID}}
+		require.NoError(t, s.CreateAgentNetworkAccessLog(ctx, entry, groups))
+	}
+	mkLog("deleted-old", deletedAccountID, old)
+	mkLog("deleted-recent", deletedAccountID, recent)
+	mkLog("keep-old", keepAccountID, old)
+	mkLog("no-settings-old", noSettingsAccountID, old)
+
+	m := &managerImpl{store: s}
+	m.cleanupAccessLogsOnce(ctx)
+
+	logIDs := func(accountID string) []string {
+		t.Helper()
+		logs, _, err := s.GetAgentNetworkAccessLogs(ctx, store.LockingStrengthNone, accountID,
+			types.AgentNetworkAccessLogFilter{Page: 1, PageSize: 50})
+		require.NoError(t, err)
+		ids := make([]string, 0, len(logs))
+		for _, l := range logs {
+			ids = append(ids, l.ID)
+		}
+		return ids
+	}
+	assert.Equal(t, []string{"deleted-recent"}, logIDs(deletedAccountID),
+		"a deleted account should have logs past the default retention swept")
+	assert.Equal(t, []string{"keep-old"}, logIDs(keepAccountID),
+		"an account with retention disabled should keep its old logs")
+	assert.Equal(t, []string{"no-settings-old"}, logIDs(noSettingsAccountID),
+		"a live account without a settings row should keep its old logs")
+}

management/internals/modules/agentnetwork/manager.go

@@ -80,6 +80,9 @@ type Manager interface {
 	ListAccessLogSessions(ctx context.Context, accountID, userID string, filter types.AgentNetworkAccessLogFilter) ([]*types.AgentNetworkAccessLogSession, int64, error)
 	GetUsageOverview(ctx context.Context, accountID, userID string, filter types.AgentNetworkAccessLogFilter, granularity types.UsageGranularity) ([]*types.AgentNetworkUsageBucket, error)
 	StartAccessLogCleanup(ctx context.Context, cleanupIntervalHours int)
+	// RemoveAccountGateway drops the account's gateway mappings from the
+	// proxies. It runs as an account deletion hook.
+	RemoveAccountGateway(ctx context.Context, accountID string) error
 	RecordConsumption(ctx context.Context, accountID string, kind types.ConsumptionDimension, dimID string, windowSeconds, tokensIn, tokensOut int64, costUSD float64) error
 	RecordAccountBudgetUsage(ctx context.Context, accountID, userID string, groupIDs []string, tokensIn, tokensOut int64, costUSD float64) error
 	RecordUsage(ctx context.Context, in RecordUsageInput) error
@@ -1350,8 +1353,8 @@ func (m *managerImpl) scopeFilterToCaller(ctx context.Context, accountID, userID
 
 // StartAccessLogCleanup launches a background sweep that periodically deletes
 // each account's agent-network access-log rows older than that account's
-// AccessLogRetentionDays. Usage records are never swept. A non-positive
-// interval defaults to 24h.
+// AccessLogRetentionDays, and the consumption counters of deleted accounts.
+// Usage records are never swept. A non-positive interval defaults to 24h.
 func (m *managerImpl) StartAccessLogCleanup(ctx context.Context, cleanupIntervalHours int) {
 	if cleanupIntervalHours <= 0 {
 		cleanupIntervalHours = 24
@@ -1362,21 +1365,40 @@ func (m *managerImpl) StartAccessLogCleanup(ctx context.Context, cleanupInterval
 		ticker := time.NewTicker(interval)
 		defer ticker.Stop()
 
-		m.cleanupAccessLogsOnce(ctx) // run once on startup
+		m.cleanupOnce(ctx) // run once on startup
 		for {
 			select {
 			case <-ctx.Done():
 				return
 			case <-ticker.C:
-				m.cleanupAccessLogsOnce(ctx)
+				m.cleanupOnce(ctx)
 			}
 		}
 	}()
 }
 
+func (m *managerImpl) cleanupOnce(ctx context.Context) {
+	m.cleanupAccessLogsOnce(ctx)
+	m.cleanupDeletedAccountConsumption(ctx)
+}
+
+// cleanupDeletedAccountConsumption deletes the consumption counters of accounts
+// that no longer exist. Best-effort: a failure is logged and retried next sweep.
+func (m *managerImpl) cleanupDeletedAccountConsumption(ctx context.Context) {
+	deleted, err := m.store.DeleteAgentNetworkConsumptionOfDeletedAccounts(ctx)
+	if err != nil {
+		log.WithContext(ctx).Warnf("agent-network consumption cleanup: %v", err)
+		return
+	}
+	if deleted > 0 {
+		log.WithContext(ctx).Infof("agent-network consumption cleanup: deleted %d counters of deleted accounts", deleted)
+	}
+}
+
 // cleanupAccessLogsOnce sweeps every account's expired access-log rows against
-// its configured retention. Best-effort: a per-account failure is logged and
-// the sweep continues.
+// its configured retention. Deleted accounts, whose settings rows went with
+// them, get the default retention. Best-effort: a per-account failure is
+// logged and the sweep continues.
 func (m *managerImpl) cleanupAccessLogsOnce(ctx context.Context) {
 	settings, err := m.store.GetAllAgentNetworkSettings(ctx, store.LockingStrengthNone)
 	if err != nil {
@@ -1384,18 +1406,31 @@ func (m *managerImpl) cleanupAccessLogsOnce(ctx context.Context) {
 		return
 	}
 	for _, s := range settings {
-		if s.AccessLogRetentionDays <= 0 {
-			continue // keep indefinitely
-		}
-		cutoff := time.Now().UTC().AddDate(0, 0, -s.AccessLogRetentionDays)
-		deleted, err := m.store.DeleteOldAgentNetworkAccessLogs(ctx, s.AccountID, cutoff)
-		if err != nil {
-			log.WithContext(ctx).Warnf("agent-network access-log cleanup for account %s: %v", s.AccountID, err)
-			continue
-		}
-		if deleted > 0 {
-			log.WithContext(ctx).Infof("agent-network access-log cleanup: deleted %d rows for account %s (retention %d days)", deleted, s.AccountID, s.AccessLogRetentionDays)
-		}
+		m.cleanupAccountAccessLogs(ctx, s.AccountID, s.AccessLogRetentionDays)
+	}
+
+	deleted, err := m.store.GetDeletedAccountIDsWithAgentNetworkAccessLogs(ctx)
+	if err != nil {
+		log.WithContext(ctx).Errorf("agent-network access-log cleanup: list deleted accounts: %v", err)
+		return
+	}
+	for _, accountID := range deleted {
+		m.cleanupAccountAccessLogs(ctx, accountID, types.DefaultAccessLogRetentionDays)
+	}
+}
+
+func (m *managerImpl) cleanupAccountAccessLogs(ctx context.Context, accountID string, retentionDays int) {
+	if retentionDays <= 0 {
+		return // keep indefinitely
+	}
+	cutoff := time.Now().UTC().AddDate(0, 0, -retentionDays)
+	deleted, err := m.store.DeleteOldAgentNetworkAccessLogs(ctx, accountID, cutoff)
+	if err != nil {
+		log.WithContext(ctx).Warnf("agent-network access-log cleanup for account %s: %v", accountID, err)
+		return
+	}
+	if deleted > 0 {
+		log.WithContext(ctx).Infof("agent-network access-log cleanup: deleted %d rows for account %s (retention %d days)", deleted, accountID, retentionDays)
 	}
 }
 
@@ -1545,6 +1580,8 @@ func (*mockManager) GetUsageOverview(_ context.Context, _, _ string, _ types.Age
 
 func (*mockManager) StartAccessLogCleanup(_ context.Context, _ int) {}
 
+func (*mockManager) RemoveAccountGateway(_ context.Context, _ string) error { return nil }
+
 func (*mockManager) RecordConsumption(_ context.Context, _ string, _ types.ConsumptionDimension, _ string, _, _, _ int64, _ float64) error {
 	return nil
 }

management/internals/modules/agentnetwork/reconcile.go

@@ -2,8 +2,10 @@ package agentnetwork
 
 import (
 	"context"
+	"fmt"
 
 	log "github.com/sirupsen/logrus"
+	goproto "google.golang.org/protobuf/proto"
 
 	rpservice "github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service"
 	"github.com/netbirdio/netbird/management/server/types"
@@ -81,18 +83,66 @@ func (m *managerImpl) reconcile(ctx context.Context, accountID string) {
 	}
 	m.reconcileMu.Unlock()
 
-	for _, entry := range creates {
-		entry.mapping.Type = proto.ProxyMappingUpdateType_UPDATE_TYPE_CREATED
-		m.proxyController.SendServiceUpdateToCluster(ctx, accountID, entry.mapping, entry.cluster)
+	m.sendMappings(ctx, accountID, creates, proto.ProxyMappingUpdateType_UPDATE_TYPE_CREATED)
+	m.sendMappings(ctx, accountID, updates, proto.ProxyMappingUpdateType_UPDATE_TYPE_MODIFIED)
+	m.sendMappings(ctx, accountID, deletes, proto.ProxyMappingUpdateType_UPDATE_TYPE_REMOVED)
+}
+
+// sendMappings sends each entry as updateType. It sends a copy: the entries'
+// mappings are shared with reconcileCache, which another reconcile or
+// RemoveAccountGateway may be reading, so they are never written.
+func (m *managerImpl) sendMappings(ctx context.Context, accountID string, entries []syntheticMapping, updateType proto.ProxyMappingUpdateType) {
+	for _, entry := range entries {
+		update := goproto.Clone(entry.mapping).(*proto.ProxyMapping)
+		update.Type = updateType
+		m.proxyController.SendServiceUpdateToCluster(ctx, accountID, update, entry.cluster)
 	}
-	for _, entry := range updates {
-		entry.mapping.Type = proto.ProxyMappingUpdateType_UPDATE_TYPE_MODIFIED
-		m.proxyController.SendServiceUpdateToCluster(ctx, accountID, entry.mapping, entry.cluster)
+}
+
+// RemoveAccountGateway tells the proxies to drop every mapping of the account's
+// gateway, so a deleted account's proxy config, provider API keys included, does
+// not linger in proxy memory until the next resync. It is an account deletion
+// hook: it runs before the account's data is removed, the last point at which
+// the mappings can be synthesised from the store. The cache alone would miss
+// them, since it is per instance and empty after a restart. If the deletion
+// then fails, the gateway stays down until the account's next change reconciles
+// it back.
+func (m *managerImpl) RemoveAccountGateway(ctx context.Context, accountID string) error {
+	if m.proxyController == nil {
+		return nil
 	}
-	for _, entry := range deletes {
-		entry.mapping.Type = proto.ProxyMappingUpdateType_UPDATE_TYPE_REMOVED
-		m.proxyController.SendServiceUpdateToCluster(ctx, accountID, entry.mapping, entry.cluster)
+
+	services, err := SynthesizeServices(ctx, m.store, accountID)
+	if err != nil {
+		return fmt.Errorf("synthesise agent network services: %w", err)
 	}
+	oidcCfg := m.proxyController.GetOIDCValidationConfig()
+	removed := make(map[string]syntheticMapping, len(services))
+	for _, svc := range services {
+		if svc == nil || svc.ID == "" {
+			continue
+		}
+		removed[svc.ID] = syntheticMapping{
+			mapping: svc.ToProtoMapping(rpservice.Delete, "", oidcCfg),
+			cluster: svc.ProxyCluster,
+		}
+	}
+
+	m.reconcileMu.Lock()
+	for id, entry := range m.reconcileCache[accountID] {
+		if _, ok := removed[id]; !ok {
+			removed[id] = entry
+		}
+	}
+	delete(m.reconcileCache, accountID)
+	m.reconcileMu.Unlock()
+
+	entries := make([]syntheticMapping, 0, len(removed))
+	for _, entry := range removed {
+		entries = append(entries, entry)
+	}
+	m.sendMappings(ctx, accountID, entries, proto.ProxyMappingUpdateType_UPDATE_TYPE_REMOVED)
+	return nil
 }
 
 // diffMappings classifies the previous→current transition for a single

management/internals/modules/agentnetwork/reconcile_test.go

@@ -2,6 +2,8 @@ package agentnetwork
 
 import (
 	"context"
+	"sync"
+	"sync/atomic"
 	"testing"
 
 	"go.uber.org/mock/gomock"
@@ -12,6 +14,7 @@ import (
 	"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/proxy"
 	"github.com/netbirdio/netbird/management/server/store"
 	"github.com/netbirdio/netbird/shared/management/proto"
+	"github.com/netbirdio/netbird/shared/management/status"
 )
 
 func newReconcileMgr(t *testing.T, ctrl *gomock.Controller) (*managerImpl, *store.MockStore, *proxy.MockController) {
@@ -287,3 +290,154 @@ func TestDiffMappings_RemovedServiceIsDeletedOnItsOwnCluster(t *testing.T) {
 		assert.Equal(t, "brave-otter.gateway.example.com", deletes[0].cluster)
 	}
 }
+
+// TestRemoveAccountGateway_EmitsRemovedFromStore — account deletion runs on an
+// instance that may never have reconciled the account, so its cache is empty.
+// The mappings are synthesised from the store, still intact before the delete,
+// and each is sent as REMOVED to the cluster that serves it.
+func TestRemoveAccountGateway_EmitsRemovedFromStore(t *testing.T) {
+	ctx := context.Background()
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	mgr, mockStore, mockProxy := newReconcileMgr(t, ctrl)
+	provider := newReconcileTestProvider()
+	policy := newReconcileTestPolicy(provider.ID, "grp-eng")
+
+	expectReconcileSynthInputs(mockStore, ctx, []*types.Provider{provider}, []*types.Policy{policy}, []*types.Guardrail{})
+	mockProxy.EXPECT().GetOIDCValidationConfig().Return(proxy.OIDCValidationConfig{})
+
+	var sent []*proto.ProxyMapping
+	mockProxy.EXPECT().
+		SendServiceUpdateToCluster(ctx, "acct-1", gomock.Any(), "eu.proxy.netbird.io").
+		Do(func(_ context.Context, _ string, m *proto.ProxyMapping, _ string) {
+			sent = append(sent, m)
+		})
+
+	require.NoError(t, mgr.RemoveAccountGateway(ctx, "acct-1"))
+
+	require.Len(t, sent, 1, "the account's one gateway mapping must be removed")
+	assert.Equal(t, proto.ProxyMappingUpdateType_UPDATE_TYPE_REMOVED, sent[0].Type, "the update must be a removal")
+	assert.Equal(t, "agent-net-svc-acct-1", sent[0].Id, "the removal must name the account's gateway service")
+}
+
+// TestRemoveAccountGateway_AlsoRemovesCachedMappings — a mapping this instance
+// last sent but the store no longer synthesises (here, one on another cluster)
+// is removed too, and the account's cache entry is cleared.
+func TestRemoveAccountGateway_AlsoRemovesCachedMappings(t *testing.T) {
+	ctx := context.Background()
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	mgr, mockStore, mockProxy := newReconcileMgr(t, ctrl)
+	mgr.reconcileCache["acct-1"] = map[string]syntheticMapping{
+		"stale-svc": {mapping: &proto.ProxyMapping{Id: "stale-svc"}, cluster: "us.proxy.netbird.io"},
+	}
+
+	// Settings but no providers: the store synthesises nothing.
+	mockStore.EXPECT().
+		GetAgentNetworkSettings(ctx, store.LockingStrengthNone, "acct-1").
+		Return(newReconcileTestSettings(), nil)
+	mockStore.EXPECT().
+		GetAccountAgentNetworkProviders(ctx, store.LockingStrengthNone, "acct-1").
+		Return([]*types.Provider{}, nil)
+	mockProxy.EXPECT().GetOIDCValidationConfig().Return(proxy.OIDCValidationConfig{})
+
+	var sent []*proto.ProxyMapping
+	mockProxy.EXPECT().
+		SendServiceUpdateToCluster(ctx, "acct-1", gomock.Any(), "us.proxy.netbird.io").
+		Do(func(_ context.Context, _ string, m *proto.ProxyMapping, _ string) {
+			sent = append(sent, m)
+		})
+
+	require.NoError(t, mgr.RemoveAccountGateway(ctx, "acct-1"))
+
+	require.Len(t, sent, 1, "the cached mapping must be removed from its own cluster")
+	assert.Equal(t, "stale-svc", sent[0].Id)
+	assert.Equal(t, proto.ProxyMappingUpdateType_UPDATE_TYPE_REMOVED, sent[0].Type)
+	mgr.reconcileMu.Lock()
+	_, present := mgr.reconcileCache["acct-1"]
+	mgr.reconcileMu.Unlock()
+	assert.False(t, present, "the deleted account's cache entry must be cleared")
+}
+
+// TestRemoveAccountGateway_SynthFailureAbortsDeletion — if the mappings cannot
+// be read, nothing is sent and the error is returned, which as an account
+// deletion hook keeps the account rather than leaving its gateway running.
+func TestRemoveAccountGateway_SynthFailureAbortsDeletion(t *testing.T) {
+	ctx := context.Background()
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	mgr, mockStore, _ := newReconcileMgr(t, ctrl)
+	mockStore.EXPECT().
+		GetAgentNetworkSettings(ctx, store.LockingStrengthNone, "acct-1").
+		Return(nil, status.Errorf(status.Internal, "store unavailable"))
+
+	assert.Error(t, mgr.RemoveAccountGateway(ctx, "acct-1"), "a failed synthesis must fail the hook")
+}
+
+func TestRemoveAccountGateway_NilProxyController_NoOp(t *testing.T) {
+	mgr := &managerImpl{reconcileCache: make(map[string]map[string]syntheticMapping)}
+	// Must not panic and must not query the store.
+	assert.NoError(t, mgr.RemoveAccountGateway(context.Background(), "acct-1"))
+}
+
+// TestReconcile_ConcurrentWithGatewayChanges — while an account's gateway
+// flaps (its policy is removed and re-added between reads), concurrent
+// reconciles and RemoveAccountGateway share the cached mappings: one caches a
+// mapping and sends it, another finds it gone and sends its removal. Run under
+// -race: neither path may write a cached mapping, only copies of it.
+func TestReconcile_ConcurrentWithGatewayChanges(t *testing.T) {
+	ctx := context.Background()
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	mgr, mockStore, mockProxy := newReconcileMgr(t, ctrl)
+	// gomock serialises every call on the controller's mutex, which would give
+	// the race detector the ordering the code under test lacks. The sends go
+	// through a fake that takes no lock.
+	mgr.proxyController = unsyncedSender{MockController: mockProxy}
+	provider := newReconcileTestProvider()
+	policy := newReconcileTestPolicy(provider.ID, "grp-eng")
+
+	var reads atomic.Int64
+	mockStore.EXPECT().GetAgentNetworkSettings(ctx, store.LockingStrengthNone, "acct-1").Return(newReconcileTestSettings(), nil).AnyTimes()
+	mockStore.EXPECT().GetAccountAgentNetworkProviders(ctx, store.LockingStrengthNone, "acct-1").Return([]*types.Provider{provider}, nil).AnyTimes()
+	mockStore.EXPECT().GetAccountAgentNetworkPolicies(ctx, store.LockingStrengthNone, "acct-1").
+		DoAndReturn(func(context.Context, store.LockingStrength, string) ([]*types.Policy, error) {
+			if reads.Add(1)%2 == 0 {
+				return []*types.Policy{}, nil
+			}
+			return []*types.Policy{policy}, nil
+		}).AnyTimes()
+	mockStore.EXPECT().GetAccountAgentNetworkGuardrails(ctx, store.LockingStrengthNone, "acct-1").Return([]*types.Guardrail{}, nil).AnyTimes()
+
+	var wg sync.WaitGroup
+	for i := 0; i < 8; i++ {
+		wg.Add(1)
+		go func(remove bool) {
+			defer wg.Done()
+			for j := 0; j < 50; j++ {
+				if remove && j%10 == 0 {
+					_ = mgr.RemoveAccountGateway(ctx, "acct-1")
+					continue
+				}
+				mgr.reconcile(ctx, "acct-1")
+			}
+		}(i == 0)
+	}
+	wg.Wait()
+}
+
+// unsyncedSender answers the calls reconcile makes on every pass without any
+// locking, so concurrent callers are not ordered by the fake itself.
+type unsyncedSender struct {
+	*proxy.MockController
+}
+
+func (unsyncedSender) GetOIDCValidationConfig() proxy.OIDCValidationConfig {
+	return proxy.OIDCValidationConfig{}
+}
+
+func (unsyncedSender) SendServiceUpdateToCluster(context.Context, string, *proto.ProxyMapping, string) {}