Migrate serializer:gob fields to serializer:json (#1855)

management/server/migration/migration.go

@@ -0,0 +1,101 @@
+package migration
+
+import (
+	"encoding/gob"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+	"gorm.io/gorm"
+)
+
+// MigrateFieldFromGobToJSON migrates a column from Gob encoding to JSON encoding.
+// T is the type of the model that contains the field to be migrated.
+// S is the type of the field to be migrated.
+func MigrateFieldFromGobToJSON[T any, S any](db *gorm.DB, fieldName string) error {
+
+	oldColumnName := fieldName
+	newColumnName := fieldName + "_tmp"
+
+	var model T
+
+	if !db.Migrator().HasTable(&model) {
+		log.Debugf("Table for %T does not exist, no migration needed", model)
+		return nil
+	}
+
+	stmt := &gorm.Statement{DB: db}
+	err := stmt.Parse(model)
+	if err != nil {
+		return fmt.Errorf("parse model: %w", err)
+	}
+	tableName := stmt.Schema.Table
+
+	var item string
+	if err := db.Model(model).Select(oldColumnName).First(&item).Error; err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			log.Debugf("No records in table %s, no migration needed", tableName)
+			return nil
+		}
+		return fmt.Errorf("fetch first record: %w", err)
+	}
+
+	var js json.RawMessage
+	var syntaxError *json.SyntaxError
+	err = json.Unmarshal([]byte(item), &js)
+	if err == nil || !errors.As(err, &syntaxError) {
+		log.Debugf("No migration needed for %s, %s", tableName, fieldName)
+		return nil
+	}
+
+	if err := db.Transaction(func(tx *gorm.DB) error {
+		if err := tx.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s TEXT", tableName, newColumnName)).Error; err != nil {
+			return fmt.Errorf("add column %s: %w", newColumnName, err)
+		}
+
+		var rows []map[string]any
+		if err := tx.Table(tableName).Select("id", oldColumnName).Find(&rows).Error; err != nil {
+			return fmt.Errorf("find rows: %w", err)
+		}
+
+		for _, row := range rows {
+			var field S
+
+			str, ok := row[oldColumnName].(string)
+			if !ok {
+				return fmt.Errorf("type assertion failed")
+			}
+			reader := strings.NewReader(str)
+
+			if err := gob.NewDecoder(reader).Decode(&field); err != nil {
+				return fmt.Errorf("gob decode error: %w", err)
+			}
+
+			jsonValue, err := json.Marshal(field)
+			if err != nil {
+				return fmt.Errorf("re-encode to JSON: %w", err)
+			}
+
+			if err := tx.Table(tableName).Where("id = ?", row["id"]).Update(newColumnName, jsonValue).Error; err != nil {
+				return fmt.Errorf("update row: %w", err)
+			}
+		}
+
+		if err := tx.Exec(fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s", tableName, oldColumnName)).Error; err != nil {
+			return fmt.Errorf("drop column %s: %w", oldColumnName, err)
+		}
+		if err := tx.Exec(fmt.Sprintf("ALTER TABLE %s RENAME COLUMN %s TO %s", tableName, newColumnName, oldColumnName)).Error; err != nil {
+			return fmt.Errorf("rename column %s to %s: %w", newColumnName, oldColumnName, err)
+		}
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	log.Infof("Migration of %s.%s from gob to json completed", tableName, fieldName)
+
+	return nil
+}

management/server/migration/migration_test.go

@@ -0,0 +1,91 @@
+package migration_test
+
+import (
+	"encoding/gob"
+	"net"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+
+	"github.com/netbirdio/netbird/management/server"
+	"github.com/netbirdio/netbird/management/server/migration"
+	"github.com/netbirdio/netbird/route"
+)
+
+func setupDatabase(t *testing.T) *gorm.DB {
+	t.Helper()
+
+	db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{
+		PrepareStmt: true,
+	})
+
+	require.NoError(t, err, "Failed to open database")
+	return db
+}
+
+func TestMigrateFieldFromGobToJSON_EmptyDB(t *testing.T) {
+	db := setupDatabase(t)
+	err := migration.MigrateFieldFromGobToJSON[server.Account, net.IPNet](db, "network_net")
+	require.NoError(t, err, "Migration should not fail for an empty database")
+}
+
+func TestMigrateFieldFromGobToJSON_WithGobData(t *testing.T) {
+	db := setupDatabase(t)
+
+	err := db.AutoMigrate(&server.Account{}, &route.Route{})
+	require.NoError(t, err, "Failed to auto-migrate tables")
+
+	_, ipnet, err := net.ParseCIDR("10.0.0.0/24")
+	require.NoError(t, err, "Failed to parse CIDR")
+
+	type network struct {
+		server.Network
+		Net net.IPNet `gorm:"serializer:gob"`
+	}
+
+	type account struct {
+		server.Account
+		Network *network `gorm:"embedded;embeddedPrefix:network_"`
+	}
+
+	err = db.Save(&account{Account: server.Account{Id: "123"}, Network: &network{Net: *ipnet}}).Error
+	require.NoError(t, err, "Failed to insert Gob data")
+
+	var gobStr string
+	err = db.Model(&server.Account{}).Select("network_net").First(&gobStr).Error
+	assert.NoError(t, err, "Failed to fetch Gob data")
+
+	err = gob.NewDecoder(strings.NewReader(gobStr)).Decode(&ipnet)
+	require.NoError(t, err, "Failed to decode Gob data")
+
+	err = migration.MigrateFieldFromGobToJSON[server.Account, net.IPNet](db, "network_net")
+	require.NoError(t, err, "Migration should not fail with Gob data")
+
+	var jsonStr string
+	db.Model(&server.Account{}).Select("network_net").First(&jsonStr)
+	assert.JSONEq(t, `{"IP":"10.0.0.0","Mask":"////AA=="}`, jsonStr, "Data should be migrated")
+}
+
+func TestMigrateFieldFromGobToJSON_WithJSONData(t *testing.T) {
+	db := setupDatabase(t)
+
+	err := db.AutoMigrate(&server.Account{}, &route.Route{})
+	require.NoError(t, err, "Failed to auto-migrate tables")
+
+	_, ipnet, err := net.ParseCIDR("10.0.0.0/24")
+	require.NoError(t, err, "Failed to parse CIDR")
+
+	err = db.Save(&server.Account{Network: &server.Network{Net: *ipnet}}).Error
+	require.NoError(t, err, "Failed to insert JSON data")
+
+	err = migration.MigrateFieldFromGobToJSON[server.Account, net.IPNet](db, "network_net")
+	require.NoError(t, err, "Migration should not fail with JSON data")
+
+	var jsonStr string
+	db.Model(&server.Account{}).Select("network_net").First(&jsonStr)
+	assert.JSONEq(t, `{"IP":"10.0.0.0","Mask":"////AA=="}`, jsonStr, "Data should be unchanged")
+}