sendnrw/netbird
2
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2026-10-29 15:59:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

[management] Remove the MySQL store engine (#7914)

Browse Source
This commit is contained in:
Bethuel Mmbaga
2026-10-20 18:50:32 +03:00
committed by GitHub
parent 569b9aa169
commit aa0f569677
51 changed files with 103 additions and 2241 deletions
Download Patch File Download Diff File Expand all files Collapse all files
management/server/migration/migration.go
+8 -46
View File
@@ -19,19 +19,11 @@ import (
"gorm.io/gorm/clause"
)
func GetColumnName(db *gorm.DB, column string) string {
if db.Name() == "mysql" {
return fmt.Sprintf("`%s`", column)
}
return column
}
// MigrateFieldFromGobToJSON migrates a column from Gob encoding to JSON encoding.
// T is the type of the model that contains the field to be migrated.
// S is the type of the field to be migrated.
func MigrateFieldFromGobToJSON[T any, S any](ctx context.Context, db *gorm.DB, fieldName string) error {
orgColumnName := fieldName
oldColumnName := GetColumnName(db, orgColumnName)
oldColumnName := fieldName
newColumnName := fieldName + "_tmp"
var model T
@@ -86,7 +78,7 @@ func MigrateFieldFromGobToJSON[T any, S any](ctx context.Context, db *gorm.DB, f
for _, row := range rows {
var field S
str, ok := row[orgColumnName].(string)
str, ok := row[oldColumnName].(string)
if !ok {
return fmt.Errorf("type assertion failed")
}
@@ -125,8 +117,7 @@ func MigrateFieldFromGobToJSON[T any, S any](ctx context.Context, db *gorm.DB, f
// MigrateNetIPFieldFromBlobToJSON migrates a Net IP column from Blob encoding to JSON encoding.
// T is the type of the model that contains the field to be migrated.
func MigrateNetIPFieldFromBlobToJSON[T any](ctx context.Context, db *gorm.DB, fieldName string, indexName string) error {
orgColumnName := fieldName
oldColumnName := GetColumnName(db, orgColumnName)
oldColumnName := fieldName
newColumnName := fieldName + "_tmp"
var model T
@@ -178,7 +169,7 @@ func MigrateNetIPFieldFromBlobToJSON[T any](ctx context.Context, db *gorm.DB, fi
for _, row := range rows {
var blobValue string
if columnValue := row[orgColumnName]; columnValue != nil {
if columnValue := row[oldColumnName]; columnValue != nil {
value, ok := columnValue.(string)
if !ok {
return fmt.Errorf("type assertion failed")
@@ -225,8 +216,7 @@ func MigrateNetIPFieldFromBlobToJSON[T any](ctx context.Context, db *gorm.DB, fi
}
func MigrateSetupKeyToHashedSetupKey[T any](ctx context.Context, db *gorm.DB) error {
orgColumnName := "key"
oldColumnName := GetColumnName(db, orgColumnName)
oldColumnName := "key"
newColumnName := "key_secret"
var model T
@@ -268,7 +258,7 @@ func MigrateSetupKeyToHashedSetupKey[T any](ctx context.Context, db *gorm.DB) er
for _, row := range rows {
var plainKey string
if columnValue := row[orgColumnName]; columnValue != nil {
if columnValue := row[oldColumnName]; columnValue != nil {
value, ok := columnValue.(string)
if !ok {
return fmt.Errorf("type assertion failed")
@@ -394,33 +384,13 @@ func CreateIndexIfNotExists[T any](ctx context.Context, db *gorm.DB, indexName s
return fmt.Errorf("failed to parse model schema: %w", err)
}
tableName := stmt.Schema.Table
dialect := db.Name()
if db.Migrator().HasIndex(&model, indexName) {
log.WithContext(ctx).Infof("index %s already exists on table %s", indexName, tableName)
return nil
}
var columnClause string
if dialect == "mysql" {
var withLength []string
for _, col := range columns {
quotedCol := fmt.Sprintf("`%s`", col)
if col == "ip" || col == "dns_label" || col == "key" {
withLength = append(withLength, fmt.Sprintf("%s(64)", quotedCol))
} else {
withLength = append(withLength, quotedCol)
}
}
columnClause = strings.Join(withLength, ", ")
} else {
columnClause = strings.Join(columns, ", ")
}
createStmt := fmt.Sprintf("CREATE UNIQUE INDEX %s ON %s (%s)", indexName, tableName, columnClause)
if dialect == "postgres" || dialect == "sqlite" {
createStmt = strings.Replace(createStmt, "CREATE UNIQUE INDEX", "CREATE UNIQUE INDEX IF NOT EXISTS", 1)
}
createStmt := fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, tableName, strings.Join(columns, ", "))
log.WithContext(ctx).Infof("executing index creation: %s", createStmt)
if err := db.Exec(createStmt).Error; err != nil {
@@ -505,14 +475,6 @@ func hasForeignKey(db *gorm.DB, table, column string) bool {
AND kcu.table_name = ?
AND kcu.column_name = ?
`, table, column).Scan(&count)
case "mysql":
db.Raw(`
SELECT COUNT(*) FROM information_schema.key_column_usage
WHERE table_schema = DATABASE()
AND table_name = ?
AND column_name = ?
AND referenced_table_name IS NOT NULL
`, table, column).Scan(&count)
default: // sqlite
type fkInfo struct {
From string
@@ -592,7 +554,7 @@ func RemoveDuplicatePeerKeys(ctx context.Context, db *gorm.DB) error {
return nil
}
keyColumn := GetColumnName(db, "key")
keyColumn := "key"
var duplicates []struct {
Key string
management/server/migration/migration_agentnetwork.go
+3 -17
View File
@@ -33,15 +33,6 @@ func (agentNetworkSettingsMigration) TableName() string { return "agent_network_
// bootstrap always wrote both, so such a row indicates corruption and the
// migration fails loudly rather than leaving an empty domain to collide with
// the unique index confusingly.
//
// The transaction is real only on sqlite and postgres, where DDL is
// transactional. MySQL implicitly commits around every ALTER TABLE, so there
// each step stands alone; what makes an interrupted run resumable on MySQL is
// that every step is guarded by the schema state it changes — the entry check
// fires while either legacy column remains, the adds skip existing columns,
// the backfill and its loud-failure check run only while the legacy cluster
// column exists (they provably completed before any drop), and each drop
// skips what is already gone.
func MigrateAgentNetworkSettingsToDomain(ctx context.Context, db *gorm.DB) error {
model := &agentNetworkSettingsMigration{}
migrator := db.Migrator()
@@ -66,14 +57,9 @@ func MigrateAgentNetworkSettingsToDomain(ctx context.Context, db *gorm.DB) error
}
if hasCluster {
concat := "subdomain || '.' || cluster"
if tx.Name() == "mysql" {
concat = "CONCAT(subdomain, '.', cluster)"
}
res := tx.Exec(fmt.Sprintf(
"UPDATE agent_network_settings SET domain = %s, proxy_address = cluster WHERE (domain IS NULL OR domain = '') AND cluster <> '' AND subdomain <> ''",
concat,
))
res := tx.Exec(
"UPDATE agent_network_settings SET domain = subdomain || '.' || cluster, proxy_address = cluster WHERE (domain IS NULL OR domain = '') AND cluster <> '' AND subdomain <> ''",
)
if res.Error != nil {
return fmt.Errorf("backfill agent_network_settings domain: %w", res.Error)
}
management/server/migration/migration_test.go
+13 -41
View File
@@ -11,7 +11,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/mysql"
"gorm.io/driver/postgres"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
@@ -32,17 +31,6 @@ func setupDatabase(t *testing.T) *gorm.DB {
var dsn string
var cleanup func()
switch os.Getenv("NETBIRD_STORE_ENGINE") {
case "mysql":
cleanup, dsn, err = testutil.CreateMysqlTestContainer()
if err != nil {
t.Fatalf("Failed to create MySQL test container: %v", err)
}
if dsn == "" {
t.Fatal("MySQL connection string is empty, ensure the test container is running")
}
db, err = gorm.Open(mysql.Open(dsn+"?charset=utf8&parseTime=True&loc=Local"), &gorm.Config{})
case "postgres":
cleanup, dsn, err = testutil.CreatePostgresTestContainer()
if err != nil {
@@ -597,9 +585,8 @@ func TestCleanupOrphanedResources_Idempotent(t *testing.T) {
}
func TestCleanupOrphanedResources_SkipsWhenForeignKeyExists(t *testing.T) {
engine := os.Getenv("NETBIRD_STORE_ENGINE")
if engine != "postgres" && engine != "mysql" {
t.Skip("FK constraint early-exit test requires postgres or mysql")
if os.Getenv("NETBIRD_STORE_ENGINE") != "postgres" {
t.Skip("FK constraint early-exit test requires postgres")
}
db := setupDatabase(t)
@@ -614,24 +601,12 @@ func TestCleanupOrphanedResources_SkipsWhenForeignKeyExists(t *testing.T) {
require.NoError(t, db.Create(&testChildWithFK{ID: "c1", ParentID: "p1"}).Error)
require.NoError(t, db.Create(&testChildWithFK{ID: "c2", ParentID: "p2"}).Error)
switch engine {
case "postgres":
require.NoError(t, db.Exec("ALTER TABLE test_children DROP CONSTRAINT fk_test_children_parent").Error)
require.NoError(t, db.Exec("DELETE FROM test_parents WHERE id = ?", "p2").Error)
require.NoError(t, db.Exec(
"ALTER TABLE test_children ADD CONSTRAINT fk_test_children_parent "+
"FOREIGN KEY (parent_id) REFERENCES test_parents(id) NOT VALID",
).Error)
case "mysql":
require.NoError(t, db.Exec("SET FOREIGN_KEY_CHECKS = 0").Error)
require.NoError(t, db.Exec("ALTER TABLE test_children DROP FOREIGN KEY fk_test_children_parent").Error)
require.NoError(t, db.Exec("DELETE FROM test_parents WHERE id = ?", "p2").Error)
require.NoError(t, db.Exec(
"ALTER TABLE test_children ADD CONSTRAINT fk_test_children_parent "+
"FOREIGN KEY (parent_id) REFERENCES test_parents(id)",
).Error)
require.NoError(t, db.Exec("SET FOREIGN_KEY_CHECKS = 1").Error)
}
require.NoError(t, db.Exec("ALTER TABLE test_children DROP CONSTRAINT fk_test_children_parent").Error)
require.NoError(t, db.Exec("DELETE FROM test_parents WHERE id = ?", "p2").Error)
require.NoError(t, db.Exec(
"ALTER TABLE test_children ADD CONSTRAINT fk_test_children_parent "+
"FOREIGN KEY (parent_id) REFERENCES test_parents(id) NOT VALID",
).Error)
err = migration.CleanupOrphanedResources[testChildWithFK, testParent](context.Background(), db, "parent_id")
require.NoError(t, err)
@@ -717,8 +692,6 @@ func TestFoldCostAggregatesIntoBuckets_SkipsAlreadyMigrated(t *testing.T) {
require.NoError(t, db.Migrator().DropTable(&agentNetworkTypes.AgentNetworkUsage{}))
require.NoError(t, db.AutoMigrate(&agentNetworkTypes.AgentNetworkUsage{}))
// Timestamp must be set explicitly: a zero time.Time serialises as
// '0000-00-00 00:00:00', which MySQL rejects under strict mode.
require.NoError(t, db.Create(&agentNetworkTypes.AgentNetworkUsage{
ID: "u1", AccountID: "acct-1", Model: "claude-sonnet-4-6",
Timestamp: time.Date(2026, 5, 5, 9, 0, 0, 0, time.UTC),
@@ -819,10 +792,9 @@ func TestMigrateAgentNetworkSettingsToDomain_FailsOnUnmigratableRow(t *testing.T
assert.Contains(t, err.Error(), "resolve them manually", "the error must tell the operator what to do")
}
// partialAgentNetworkSettings models the one non-atomic state a MySQL run can
// be interrupted in: DDL auto-commits there, so a crash between the two legacy
// column drops leaves subdomain behind while cluster (and the completed
// backfill) are already committed.
// partialAgentNetworkSettings models a schema left between the two legacy
// column drops: subdomain is still present while cluster is gone and the
// backfill has already completed.
type partialAgentNetworkSettings struct {
AccountID string `gorm:"primaryKey"`
Subdomain string
@@ -832,8 +804,8 @@ type partialAgentNetworkSettings struct {
func (partialAgentNetworkSettings) TableName() string { return "agent_network_settings" }
// TestMigrateAgentNetworkSettingsToDomain_ResumesAfterPartialDrop pins MySQL
// resumability: a rerun over the interrupted state must remove the leftover
// TestMigrateAgentNetworkSettingsToDomain_ResumesAfterPartialDrop pins
// resumability: a rerun over the partial state must remove the leftover
// subdomain column without re-running the backfill (the cluster column that
// feeds it is gone) and without touching the migrated values.
func TestMigrateAgentNetworkSettingsToDomain_ResumesAfterPartialDrop(t *testing.T) {