From 72a23bcdf459cf4e4669fa71c7c9fadbb6b4960f Mon Sep 17 00:00:00 2001 From: mlsmaycon Date: Thu, 13 Aug 2026 18:52:11 +0000 Subject: [PATCH] [test] Stop leaking management servers in the integration harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BuildApiBlackBoxWithDBState[AndPeerChannel] built the account manager, telemetry metrics, and API handler on context.Background() and registered no cleanup. Every background loop those components start (AccountRequestBuffer.processGetAccountRequests, the telemetry P95 flushers, PATUsageTracker.reportLoop, APIRateLimiter.cleanupLoop, proxy service cleanup, cache janitors) exits only on ctx.Done(), so on a never-cancelled context they ran forever and piled up across the package — along with each server's sql.DB connection pool. Over a package run that builds ~150 servers this exhausts DB connections against the real Postgres/MySQL backends, so per-test store setup crawls until the suite trips the 20m go-test timeout (seen as timeouts in Management/Integration (postgres) and Management/Unit (mysql); the in-process sqlite variants finish before it bites). Give each helper a cancellable context tied to t.Cleanup(cancel) so the manager and its goroutines/pools wind down when the test ends. Test-only change; production already cancels the server context on shutdown. --- .../testing/testing_tools/channel/channel.go | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/management/server/http/testing/testing_tools/channel/channel.go b/management/server/http/testing/testing_tools/channel/channel.go index 8b05b2ddf..28362427a 100644 --- a/management/server/http/testing/testing_tools/channel/channel.go +++ b/management/server/http/testing/testing_tools/channel/channel.go @@ -59,13 +59,21 @@ func BuildApiBlackBoxWithDBState(t testing_tools.TB, sqlFile string, expectedPee } t.Cleanup(cleanup) - metrics, err := telemetry.NewDefaultAppMetrics(context.Background()) + // Bound the background loops these managers start (account request buffer, + // telemetry P95 flushers, PAT usage tracker, API rate limiter, proxy service + // cleanup, cache janitors, DB connection pools) to the test's lifetime. On + // context.Background() they never stop and accumulate across the package, + // exhausting DB connections until the suite hits the 20m test timeout. + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + metrics, err := telemetry.NewDefaultAppMetrics(ctx) if err != nil { t.Fatalf("Failed to create metrics: %v", err) } peersUpdateManager := update_channel.NewPeersUpdateManager(nil) - updMsg := peersUpdateManager.CreateChannel(context.Background(), testing_tools.TestPeerId) + updMsg := peersUpdateManager.CreateChannel(ctx, testing_tools.TestPeerId) done := make(chan struct{}) if validateUpdate { go func() { @@ -88,8 +96,6 @@ func BuildApiBlackBoxWithDBState(t testing_tools.TB, sqlFile string, expectedPee jobManager := job.NewJobManager(nil, store, peersManager) - ctx := context.Background() - cacheStore, err := nbcache.NewStore(ctx, 100*time.Millisecond, 300*time.Millisecond, 100) if err != nil { t.Fatalf("Failed to create cache store: %v", err) @@ -137,7 +143,7 @@ func BuildApiBlackBoxWithDBState(t testing_tools.TB, sqlFile string, expectedPee zoneRecordsManager := recordsManager.NewManager(store, am, permissionsManager) apiRouter := mux.NewRouter().PathPrefix("/api").Subrouter() - apiHandler, err := http2.NewAPIHandler(context.Background(), apiRouter, am, networksManager, resourcesManager, routersManager, groupsManager, geoMock, authManagerMock, metrics, permissionsManager, settingsManager, customZonesManager, zoneRecordsManager, networkMapController, nil, serviceManager, nil, nil, nil, nil, nil, nil, nil) + apiHandler, err := http2.NewAPIHandler(ctx, apiRouter, am, networksManager, resourcesManager, routersManager, groupsManager, geoMock, authManagerMock, metrics, permissionsManager, settingsManager, customZonesManager, zoneRecordsManager, networkMapController, nil, serviceManager, nil, nil, nil, nil, nil, nil, nil) if err != nil { t.Fatalf("Failed to create API handler: %v", err) } @@ -200,13 +206,21 @@ func BuildApiBlackBoxWithDBStateAndPeerChannel(t testing_tools.TB, sqlFile strin } t.Cleanup(cleanup) - metrics, err := telemetry.NewDefaultAppMetrics(context.Background()) + // Bound the background loops these managers start (account request buffer, + // telemetry P95 flushers, PAT usage tracker, API rate limiter, proxy service + // cleanup, cache janitors, DB connection pools) to the test's lifetime. On + // context.Background() they never stop and accumulate across the package, + // exhausting DB connections until the suite hits the 20m test timeout. + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + metrics, err := telemetry.NewDefaultAppMetrics(ctx) if err != nil { t.Fatalf("Failed to create metrics: %v", err) } peersUpdateManager := update_channel.NewPeersUpdateManager(nil) - updMsg := peersUpdateManager.CreateChannel(context.Background(), testing_tools.TestPeerId) + updMsg := peersUpdateManager.CreateChannel(ctx, testing_tools.TestPeerId) geoMock := &geolocation.Mock{} validatorMock := server.MockIntegratedValidator{} @@ -218,8 +232,6 @@ func BuildApiBlackBoxWithDBStateAndPeerChannel(t testing_tools.TB, sqlFile strin jobManager := job.NewJobManager(nil, store, peersManager) - ctx := context.Background() - cacheStore, err := nbcache.NewStore(ctx, 100*time.Millisecond, 300*time.Millisecond, 100) if err != nil { t.Fatalf("Failed to create cache store: %v", err) @@ -267,7 +279,7 @@ func BuildApiBlackBoxWithDBStateAndPeerChannel(t testing_tools.TB, sqlFile strin zoneRecordsManager := recordsManager.NewManager(store, am, permissionsManager) apiRouter := mux.NewRouter().PathPrefix("/api").Subrouter() - apiHandler, err := http2.NewAPIHandler(context.Background(), apiRouter, am, networksManager, resourcesManager, routersManager, groupsManager, geoMock, authManagerMock, metrics, permissionsManager, settingsManager, customZonesManager, zoneRecordsManager, networkMapController, nil, serviceManager, nil, nil, nil, nil, nil, nil, nil) + apiHandler, err := http2.NewAPIHandler(ctx, apiRouter, am, networksManager, resourcesManager, routersManager, groupsManager, geoMock, authManagerMock, metrics, permissionsManager, settingsManager, customZonesManager, zoneRecordsManager, networkMapController, nil, serviceManager, nil, nil, nil, nil, nil, nil, nil) if err != nil { t.Fatalf("Failed to create API handler: %v", err) }