diff --git a/management/server/migration/migration.go b/management/server/migration/migration.go index ba31ee459..9776418ad 100644 --- a/management/server/migration/migration.go +++ b/management/server/migration/migration.go @@ -1,10 +1,12 @@ package migration import ( + "database/sql" "encoding/gob" "encoding/json" "errors" "fmt" + "net" "strings" log "github.com/sirupsen/logrus" @@ -99,3 +101,104 @@ func MigrateFieldFromGobToJSON[T any, S any](db *gorm.DB, fieldName string) erro return nil } + +// MigrateNetIPFieldFromBlobToJSON migrates a Net IP column from Blob encoding to JSON encoding. +// T is the type of the model that contains the field to be migrated. +func MigrateNetIPFieldFromBlobToJSON[T any](db *gorm.DB, fieldName string, indexName string) error { + oldColumnName := fieldName + newColumnName := fieldName + "_tmp" + + var model T + + if !db.Migrator().HasTable(&model) { + log.Printf("Table for %T does not exist, no migration needed", model) + return nil + } + + stmt := &gorm.Statement{DB: db} + err := stmt.Parse(&model) + if err != nil { + return fmt.Errorf("parse model: %w", err) + } + tableName := stmt.Schema.Table + + var item sql.NullString + if err := db.Model(&model).Select(oldColumnName).First(&item).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + log.Printf("No records in table %s, no migration needed", tableName) + return nil + } + return fmt.Errorf("fetch first record: %w", err) + } + + if item.Valid { + var js json.RawMessage + var syntaxError *json.SyntaxError + err = json.Unmarshal([]byte(item.String), &js) + if err == nil || !errors.As(err, &syntaxError) { + log.Debugf("No migration needed for %s, %s", tableName, fieldName) + return nil + } + } + + if err := db.Transaction(func(tx *gorm.DB) error { + if err := tx.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s TEXT", tableName, newColumnName)).Error; err != nil { + return fmt.Errorf("add column %s: %w", newColumnName, err) + } + + var rows []map[string]any + if err := tx.Table(tableName).Select("id", oldColumnName).Find(&rows).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + log.Printf("No records in table %s, no migration needed", tableName) + return nil + } + return fmt.Errorf("find rows: %w", err) + } + + for _, row := range rows { + var blobValue string + if columnValue := row[oldColumnName]; columnValue != nil { + value, ok := columnValue.(string) + if !ok { + return fmt.Errorf("type assertion failed") + } + blobValue = value + } + + columnIpValue := net.IP(blobValue) + if net.ParseIP(columnIpValue.String()) == nil { + log.Debugf("failed to parse %s as ip, fallback to ipv6 loopback", oldColumnName) + columnIpValue = net.IPv6loopback + } + + jsonValue, err := json.Marshal(columnIpValue) + if err != nil { + return fmt.Errorf("re-encode to JSON: %w", err) + } + + if err := tx.Table(tableName).Where("id = ?", row["id"]).Update(newColumnName, jsonValue).Error; err != nil { + return fmt.Errorf("update row: %w", err) + } + } + + if indexName != "" { + if err := tx.Migrator().DropIndex(&model, indexName); err != nil { + return fmt.Errorf("drop index %s: %w", indexName, err) + } + } + + if err := tx.Exec(fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", tableName, oldColumnName)).Error; err != nil { + return fmt.Errorf("drop column %s: %w", oldColumnName, err) + } + if err := tx.Exec(fmt.Sprintf("ALTER TABLE %s RENAME COLUMN %s TO %s", tableName, newColumnName, oldColumnName)).Error; err != nil { + return fmt.Errorf("rename column %s to %s: %w", newColumnName, oldColumnName, err) + } + return nil + }); err != nil { + return err + } + + log.Printf("Migration of %s.%s from blob to json completed", tableName, fieldName) + + return nil +} diff --git a/management/server/migration/migration_test.go b/management/server/migration/migration_test.go index 4bef41b86..45757e9d6 100644 --- a/management/server/migration/migration_test.go +++ b/management/server/migration/migration_test.go @@ -13,6 +13,7 @@ import ( "github.com/netbirdio/netbird/management/server" "github.com/netbirdio/netbird/management/server/migration" + nbpeer "github.com/netbirdio/netbird/management/server/peer" "github.com/netbirdio/netbird/route" ) @@ -89,3 +90,72 @@ func TestMigrateFieldFromGobToJSON_WithJSONData(t *testing.T) { db.Model(&server.Account{}).Select("network_net").First(&jsonStr) assert.JSONEq(t, `{"IP":"10.0.0.0","Mask":"////AA=="}`, jsonStr, "Data should be unchanged") } + +func TestMigrateNetIPFieldFromBlobToJSON_EmptyDB(t *testing.T) { + db := setupDatabase(t) + err := migration.MigrateNetIPFieldFromBlobToJSON[nbpeer.Peer](db, "ip", "idx_peers_account_id_ip") + require.NoError(t, err, "Migration should not fail for an empty database") +} + +func TestMigrateNetIPFieldFromBlobToJSON_WithBlobData(t *testing.T) { + db := setupDatabase(t) + + err := db.AutoMigrate(&server.Account{}, &nbpeer.Peer{}) + require.NoError(t, err, "Failed to auto-migrate tables") + + type location struct { + nbpeer.Location + ConnectionIP net.IP + } + + type peer struct { + nbpeer.Peer + Location location `gorm:"embedded;embeddedPrefix:location_"` + } + + type account struct { + server.Account + Peers []peer `gorm:"foreignKey:AccountID;references:id"` + } + + err = db.Save(&account{ + Account: server.Account{Id: "123"}, + Peers: []peer{ + {Location: location{ConnectionIP: net.IP{10, 0, 0, 1}}}, + }}, + ).Error + require.NoError(t, err, "Failed to insert blob data") + + var blobValue string + err = db.Model(&nbpeer.Peer{}).Select("location_connection_ip").First(&blobValue).Error + assert.NoError(t, err, "Failed to fetch blob data") + + err = migration.MigrateNetIPFieldFromBlobToJSON[nbpeer.Peer](db, "location_connection_ip", "") + require.NoError(t, err, "Migration should not fail with net.IP blob data") + + var jsonStr string + db.Model(&nbpeer.Peer{}).Select("location_connection_ip").First(&jsonStr) + assert.JSONEq(t, `"10.0.0.1"`, jsonStr, "Data should be migrated") +} + +func TestMigrateNetIPFieldFromBlobToJSON_WithJSONData(t *testing.T) { + db := setupDatabase(t) + + err := db.AutoMigrate(&server.Account{}, &nbpeer.Peer{}) + require.NoError(t, err, "Failed to auto-migrate tables") + + err = db.Save(&server.Account{ + Id: "1234", + PeersG: []nbpeer.Peer{ + {Location: nbpeer.Location{ConnectionIP: net.IP{10, 0, 0, 1}}}, + }}, + ).Error + require.NoError(t, err, "Failed to insert JSON data") + + err = migration.MigrateNetIPFieldFromBlobToJSON[nbpeer.Peer](db, "location_connection_ip", "") + require.NoError(t, err, "Migration should not fail with net.IP JSON data") + + var jsonStr string + db.Model(&nbpeer.Peer{}).Select("location_connection_ip").First(&jsonStr) + assert.JSONEq(t, `"10.0.0.1"`, jsonStr, "Data should be unchanged") +} diff --git a/management/server/peer/peer.go b/management/server/peer/peer.go index e62843180..0d0eabb2c 100644 --- a/management/server/peer/peer.go +++ b/management/server/peer/peer.go @@ -13,13 +13,13 @@ type Peer struct { // ID is an internal ID of the peer ID string `gorm:"primaryKey"` // AccountID is a reference to Account that this object belongs - AccountID string `json:"-" gorm:"index;uniqueIndex:idx_peers_account_id_ip"` + AccountID string `json:"-" gorm:"index"` // WireGuard public key Key string `gorm:"index"` // A setup key this peer was registered with SetupKey string // IP address of the Peer - IP net.IP `gorm:"uniqueIndex:idx_peers_account_id_ip"` + IP net.IP `gorm:"serializer:json"` // Meta is a Peer system meta data Meta PeerSystemMeta `gorm:"embedded;embeddedPrefix:meta_"` // Name is peer's name (machine name) @@ -61,7 +61,7 @@ type PeerStatus struct { //nolint:revive // Location is a geo location information of a Peer based on public connection IP type Location struct { - ConnectionIP net.IP // from grpc peer or reverse proxy headers depends on setup + ConnectionIP net.IP `gorm:"serializer:json"` // from grpc peer or reverse proxy headers depends on setup CountryCode string CityName string GeoNameID uint // city level geoname id diff --git a/management/server/sqlite_store.go b/management/server/sqlite_store.go index 65281a4f8..853816fd3 100644 --- a/management/server/sqlite_store.go +++ b/management/server/sqlite_store.go @@ -571,13 +571,17 @@ func getMigrations() []migrationFunc { func(db *gorm.DB) error { return migration.MigrateFieldFromGobToJSON[Account, net.IPNet](db, "network_net") }, - func(db *gorm.DB) error { return migration.MigrateFieldFromGobToJSON[route.Route, netip.Prefix](db, "network") }, - func(db *gorm.DB) error { return migration.MigrateFieldFromGobToJSON[route.Route, []string](db, "peer_groups") }, + func(db *gorm.DB) error { + return migration.MigrateNetIPFieldFromBlobToJSON[nbpeer.Peer](db, "location_connection_ip", "") + }, + func(db *gorm.DB) error { + return migration.MigrateNetIPFieldFromBlobToJSON[nbpeer.Peer](db, "ip", "idx_peers_account_id_ip") + }, } } diff --git a/management/server/sqlite_store_test.go b/management/server/sqlite_store_test.go index cc033e61f..8758e3b07 100644 --- a/management/server/sqlite_store_test.go +++ b/management/server/sqlite_store_test.go @@ -519,15 +519,29 @@ func TestMigrate(t *testing.T) { Net net.IPNet `gorm:"serializer:gob"` } + type location struct { + nbpeer.Location + ConnectionIP net.IP + } + + type peer struct { + nbpeer.Peer + Location location `gorm:"embedded;embeddedPrefix:location_"` + } + type account struct { Account Network *network `gorm:"embedded;embeddedPrefix:network_"` + Peers []peer `gorm:"foreignKey:AccountID;references:id"` } act := &account{ Network: &network{ Net: *ipnet, }, + Peers: []peer{ + {Location: location{ConnectionIP: net.IP{10, 0, 0, 1}}}, + }, } err = store.db.Save(act).Error