mirror of
https://github.com/netbirdio/netbird.git
synced 2026-10-07 22:19:08 +02:00
[management] Add a go test -json summarizer and log test store setup time
The Management / Unit (mysql) job hit the 20 minute go test budget in management/server and left only a goroutine dump of several thousand leaked goroutines, with the "running tests:" header far out of reach of the job log viewer. The last live test goroutine was runnable, not blocked: the suite was still progressing when the alarm fired. tools/gotestsummary reads a go test -json stream and prints one line per finished test, the output of failed tests, the head of a package panic (the timeout header and the running tests list), tests that never reported a result, and the slowest tests per package. NewTestStoreFromSQL now logs how long the sqlite seed store and the engine under test took to come up; the summarizer attributes those to the owning test so the per-test store cost on mysql and postgres is visible in CI. Two leaks made the dump unreadable and are fixed here: the sqlite seed store was never closed once a mysql or postgres store replaced it, and buildTestManager passed context.Background() to the metrics, caches and controllers, so every test left its P95 flushers and cleanup loops running until the process exited.
This commit is contained in:
@@ -3516,7 +3516,13 @@ func buildTestManager(t testing.TB, store store.Store, nmdataStore *networkmapdb
|
||||
|
||||
eventStore := &activity.InMemoryEventStore{}
|
||||
|
||||
metrics, err := telemetry.NewDefaultAppMetrics(context.Background())
|
||||
// Everything built here watches this context; cancelling it on cleanup stops
|
||||
// the metrics flushers, caches and controllers instead of leaking them for
|
||||
// the rest of the package run.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
|
||||
metrics, err := telemetry.NewDefaultAppMetrics(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -3535,8 +3541,6 @@ func buildTestManager(t testing.TB, store store.Store, nmdataStore *networkmapdb
|
||||
Return(nil).
|
||||
AnyTimes()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
cacheStore, err := cache.NewStore(ctx, 100*time.Millisecond, 300*time.Millisecond, 100)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
|
||||
@@ -685,6 +685,7 @@ func getMigrationsPostAuto(ctx context.Context) []migrationFunc {
|
||||
// NewTestStoreFromSQL is only used in tests. It will create a test database base of the store engine set in env.
|
||||
// Optionally it can load a SQL file to the database. If the filename is empty it will return an empty database
|
||||
func NewTestStoreFromSQL(ctx context.Context, filename string, dataDir string) (Store, func(), error) {
|
||||
start := time.Now()
|
||||
kind := getStoreEngineFromEnv()
|
||||
if kind == "" {
|
||||
kind = types.SqliteStoreEngine
|
||||
@@ -722,10 +723,15 @@ func NewTestStoreFromSQL(ctx context.Context, filename string, dataDir string) (
|
||||
var sqlStore Store
|
||||
var cleanup func()
|
||||
|
||||
sqliteReady := time.Now()
|
||||
maxRetries := 2
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
sqlStore, cleanup, err = getSqlStoreEngine(ctx, store, kind)
|
||||
if err == nil {
|
||||
// Parsed by tools/gotestsummary to attribute store setup time per test.
|
||||
log.WithContext(ctx).Infof("test store created: engine=%s total=%s sqlite=%s engine_setup=%s",
|
||||
kind, time.Since(start).Round(time.Millisecond), sqliteReady.Sub(start).Round(time.Millisecond),
|
||||
time.Since(sqliteReady).Round(time.Millisecond))
|
||||
return sqlStore, cleanup, nil
|
||||
}
|
||||
if i < maxRetries-1 {
|
||||
@@ -758,14 +764,15 @@ func addAllGroupToAccount(ctx context.Context, store Store) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func getSqlStoreEngine(ctx context.Context, store *SqlStore, kind types.Engine) (Store, func(), error) {
|
||||
func getSqlStoreEngine(ctx context.Context, sqliteStore *SqlStore, kind types.Engine) (Store, func(), error) {
|
||||
store := sqliteStore
|
||||
var cleanup func()
|
||||
var err error
|
||||
switch kind {
|
||||
case types.PostgresStoreEngine:
|
||||
store, cleanup, err = newReusedPostgresStore(ctx, store, kind)
|
||||
store, cleanup, err = newReusedPostgresStore(ctx, sqliteStore, kind)
|
||||
case types.MysqlStoreEngine:
|
||||
store, cleanup, err = newReusedMysqlStore(ctx, store, kind)
|
||||
store, cleanup, err = newReusedMysqlStore(ctx, sqliteStore, kind)
|
||||
default:
|
||||
cleanup = func() {
|
||||
// sqlite doesn't need to be cleaned up
|
||||
@@ -781,6 +788,11 @@ func getSqlStoreEngine(ctx context.Context, store *SqlStore, kind types.Engine)
|
||||
if store.pool != nil {
|
||||
store.pool.Close()
|
||||
}
|
||||
if store != sqliteStore {
|
||||
// The sqlite store only seeded the engine under test; without this
|
||||
// every test leaks its connection and the opener goroutines.
|
||||
sqliteStore.Close(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
return store, closeConnection, nil
|
||||
|
||||
Reference in New Issue
Block a user