[management] Add a go test -json summarizer and log test store setup time

The Management / Unit (mysql) job hit the 20 minute go test budget in
management/server and left only a goroutine dump of several thousand
leaked goroutines, with the "running tests:" header far out of reach of
the job log viewer. The last live test goroutine was runnable, not
blocked: the suite was still progressing when the alarm fired.

tools/gotestsummary reads a go test -json stream and prints one line
per finished test, the output of failed tests, the head of a package
panic (the timeout header and the running tests list), tests that never
reported a result, and the slowest tests per package.

NewTestStoreFromSQL now logs how long the sqlite seed store and the
engine under test took to come up; the summarizer attributes those to
the owning test so the per-test store cost on mysql and postgres is
visible in CI.

Two leaks made the dump unreadable and are fixed here: the sqlite seed
store was never closed once a mysql or postgres store replaced it, and
buildTestManager passed context.Background() to the metrics, caches and
controllers, so every test left its P95 flushers and cleanup loops
running until the process exited.
This commit is contained in:
mlsmaycon
2026-09-12 12:00:37 +00:00
parent 973173be9c
commit ba9b226b4b
3 changed files with 355 additions and 6 deletions
+7 -3
View File
@@ -3516,7 +3516,13 @@ func buildTestManager(t testing.TB, store store.Store, nmdataStore *networkmapdb
eventStore := &activity.InMemoryEventStore{}
metrics, err := telemetry.NewDefaultAppMetrics(context.Background())
// Everything built here watches this context; cancelling it on cleanup stops
// the metrics flushers, caches and controllers instead of leaking them for
// the rest of the package run.
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
metrics, err := telemetry.NewDefaultAppMetrics(ctx)
if err != nil {
return nil, nil, err
}
@@ -3535,8 +3541,6 @@ func buildTestManager(t testing.TB, store store.Store, nmdataStore *networkmapdb
Return(nil).
AnyTimes()
ctx := context.Background()
cacheStore, err := cache.NewStore(ctx, 100*time.Millisecond, 300*time.Millisecond, 100)
if err != nil {
return nil, nil, err
+15 -3
View File
@@ -685,6 +685,7 @@ func getMigrationsPostAuto(ctx context.Context) []migrationFunc {
// NewTestStoreFromSQL is only used in tests. It will create a test database base of the store engine set in env.
// Optionally it can load a SQL file to the database. If the filename is empty it will return an empty database
func NewTestStoreFromSQL(ctx context.Context, filename string, dataDir string) (Store, func(), error) {
start := time.Now()
kind := getStoreEngineFromEnv()
if kind == "" {
kind = types.SqliteStoreEngine
@@ -722,10 +723,15 @@ func NewTestStoreFromSQL(ctx context.Context, filename string, dataDir string) (
var sqlStore Store
var cleanup func()
sqliteReady := time.Now()
maxRetries := 2
for i := 0; i < maxRetries; i++ {
sqlStore, cleanup, err = getSqlStoreEngine(ctx, store, kind)
if err == nil {
// Parsed by tools/gotestsummary to attribute store setup time per test.
log.WithContext(ctx).Infof("test store created: engine=%s total=%s sqlite=%s engine_setup=%s",
kind, time.Since(start).Round(time.Millisecond), sqliteReady.Sub(start).Round(time.Millisecond),
time.Since(sqliteReady).Round(time.Millisecond))
return sqlStore, cleanup, nil
}
if i < maxRetries-1 {
@@ -758,14 +764,15 @@ func addAllGroupToAccount(ctx context.Context, store Store) error {
return nil
}
func getSqlStoreEngine(ctx context.Context, store *SqlStore, kind types.Engine) (Store, func(), error) {
func getSqlStoreEngine(ctx context.Context, sqliteStore *SqlStore, kind types.Engine) (Store, func(), error) {
store := sqliteStore
var cleanup func()
var err error
switch kind {
case types.PostgresStoreEngine:
store, cleanup, err = newReusedPostgresStore(ctx, store, kind)
store, cleanup, err = newReusedPostgresStore(ctx, sqliteStore, kind)
case types.MysqlStoreEngine:
store, cleanup, err = newReusedMysqlStore(ctx, store, kind)
store, cleanup, err = newReusedMysqlStore(ctx, sqliteStore, kind)
default:
cleanup = func() {
// sqlite doesn't need to be cleaned up
@@ -781,6 +788,11 @@ func getSqlStoreEngine(ctx context.Context, store *SqlStore, kind types.Engine)
if store.pool != nil {
store.pool.Close()
}
if store != sqliteStore {
// The sqlite store only seeded the engine under test; without this
// every test leaks its connection and the opener goroutines.
sqliteStore.Close(ctx)
}
}
return store, closeConnection, nil