fix: export from SQLite fails to import into Postgres because of mismatched schemas

backend/internal/utils/db_schema_parity_test.go

@@ -0,0 +1,52 @@
+//go:build unit
+
+package utils_test
+
+import (
+	"maps"
+	"slices"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/pocket-id/pocket-id/backend/internal/utils"
+	testutils "github.com/pocket-id/pocket-id/backend/internal/utils/testing"
+)
+
+// TestDBSchemaParity ensures the SQLite and Postgres migrations produce schemas that encode every column the same way in a data export
+// Without this, an export taken from one database provider fails to import into the other (see https://github.com/pocket-id/pocket-id/issues/1603 and https://github.com/pocket-id/pocket-id/issues/1815)
+// If this test fails after adding a migration, pick column types for both databases that map to the same utils.DBExportKind
+func TestDBSchemaParity(t *testing.T) {
+	sqliteSchema, err := utils.LoadDBSchemaTypes(testutils.NewDatabaseForTest(t))
+	require.NoError(t, err)
+	postgresSchema, err := utils.LoadDBSchemaTypes(testutils.NewPostgresDatabaseForTest(t))
+	require.NoError(t, err)
+
+	// The migration bookkeeping table is managed by golang-migrate and never exported
+	delete(sqliteSchema, "schema_migrations")
+	delete(postgresSchema, "schema_migrations")
+
+	// Both databases must have the same tables
+	require.ElementsMatch(t, slices.Collect(maps.Keys(sqliteSchema)), slices.Collect(maps.Keys(postgresSchema)), "SQLite and Postgres must have the same tables")
+
+	for table, sqliteColumns := range sqliteSchema {
+		postgresColumns := postgresSchema[table]
+
+		// Both databases must have the same columns in each table
+		if !assert.ElementsMatchf(t, slices.Collect(maps.Keys(sqliteColumns)), slices.Collect(maps.Keys(postgresColumns)), "SQLite and Postgres must have the same columns in table %q", table) {
+			continue
+		}
+
+		// Each column must be encoded the same way in an export and accept the same values
+		for column, sqliteColumn := range sqliteColumns {
+			postgresColumn := postgresColumns[column]
+			assert.Equalf(t, sqliteColumn.ExportKind(), postgresColumn.ExportKind(),
+				"column %s.%s is exported differently: SQLite type %q is %q, Postgres type %q is %q",
+				table, column, sqliteColumn.Name, sqliteColumn.ExportKind(), postgresColumn.Name, postgresColumn.ExportKind())
+			assert.Equalf(t, sqliteColumn.Nullable, postgresColumn.Nullable,
+				"column %s.%s has a different nullability: SQLite nullable=%t, Postgres nullable=%t",
+				table, column, sqliteColumn.Nullable, postgresColumn.Nullable)
+		}
+	}
+}

backend/internal/utils/db_util.go

@@ -37,6 +37,38 @@ type DBSchemaColumn struct {
 	Name     string
 	Nullable bool
 }
+
+// DBExportKind is how the values of a column are encoded in a data export
+type DBExportKind string
+
+const (
+	DBExportKindString   DBExportKind = "string"
+	DBExportKindBool     DBExportKind = "bool"
+	DBExportKindInteger  DBExportKind = "integer"
+	DBExportKindDateTime DBExportKind = "datetime"
+	// DBExportKindBytes values are stored as base64-encoded strings
+	DBExportKindBytes DBExportKind = "bytes"
+)
+
+// ExportKind returns how the values of the column are encoded in a data export
+// An export from one database provider can only be imported into another if every column has the same export kind on both
+func (c DBSchemaColumn) ExportKind() DBExportKind {
+	switch c.Name {
+	case "boolean", "bool":
+		return DBExportKindBool
+	case "integer", "int", "bigint":
+		return DBExportKindInteger
+	case "timestamp", "timestamptz", "timestamp with time zone", "datetime":
+		return DBExportKindDateTime
+	case "blob", "bytea", "jsonb":
+		// jsonb is treated as binary too
+		return DBExportKindBytes
+	default:
+		// Everything else is treated as a string, including the "numeric" type
+		return DBExportKindString
+	}
+}
+
 type DBSchemaTableTypes = map[string]DBSchemaColumn
 type DBSchemaTypes = map[string]DBSchemaTableTypes
 

backend/internal/utils/testing/database.go

@@ -5,20 +5,26 @@
 package testing
 
 import (
+	"crypto/rand"
 	"errors"
 	"log/slog"
+	"net/url"
+	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
 	_ "github.com/golang-migrate/migrate/v4/source/file"
 
 	"github.com/golang-migrate/migrate/v4"
+	postgresMigrate "github.com/golang-migrate/migrate/v4/database/postgres"
 	sqliteMigrate "github.com/golang-migrate/migrate/v4/database/sqlite3"
 	"github.com/golang-migrate/migrate/v4/source/iofs"
 	sqlitekit "github.com/italypaleale/go-sql-utils/sqlite"
 	"github.com/libtnb/sqlite"
 	"github.com/stretchr/testify/require"
+	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
 
@@ -62,6 +68,61 @@ func NewConcurrentDatabaseForTest(t *testing.T) *gorm.DB {
 	return db
 }
 
+// NewPostgresDatabaseForTest returns a new instance of GORM connected to a fresh Postgres database with all Postgres migrations applied.
+// The server is taken from the POCKET_ID_TEST_POSTGRES_URL environment variable, and a new database is created on it for each test and dropped afterwards.
+// When the variable is not set the test is skipped, unless POCKET_ID_TEST_POSTGRES_REQUIRED is "true", which CI sets so these tests can never be skipped silently there.
+func NewPostgresDatabaseForTest(t *testing.T) *gorm.DB {
+	t.Helper()
+
+	serverURL := os.Getenv("POCKET_ID_TEST_POSTGRES_URL")
+	if serverURL == "" {
+		if os.Getenv("POCKET_ID_TEST_POSTGRES_REQUIRED") == "true" {
+			t.Fatal("POCKET_ID_TEST_POSTGRES_URL must be set when POCKET_ID_TEST_POSTGRES_REQUIRED is true")
+		}
+		t.Skip("POCKET_ID_TEST_POSTGRES_URL is not set")
+	}
+
+	// Create a database that only this test uses, so tests can run in parallel against the same server
+	admin, err := gorm.Open(postgres.Open(serverURL), newTestGormConfig(t))
+	require.NoError(t, err, "Failed to connect to the Postgres test server")
+	adminDB, err := admin.DB()
+	require.NoError(t, err, "Failed to get sql.DB")
+	dbName := "pocket_id_test_" + strings.ToLower(rand.Text())
+	require.NoError(t, admin.Exec(`CREATE DATABASE "`+dbName+`"`).Error, "Failed to create the Postgres test database")
+
+	// Drop the database once the test is done, after its own connections have been closed by the cleanup registered below
+	t.Cleanup(func() {
+		require.NoError(t, admin.Exec(`DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`).Error, "Failed to drop the Postgres test database")
+		require.NoError(t, adminDB.Close(), "Failed to close the Postgres test server connection")
+	})
+
+	// Connect to the new database by swapping the database name in the server URL
+	u, err := url.Parse(serverURL)
+	require.NoError(t, err, "Failed to parse POCKET_ID_TEST_POSTGRES_URL")
+	u.Path = "/" + dbName
+	db, err := gorm.Open(postgres.Open(u.String()), newTestGormConfig(t))
+	require.NoError(t, err, "Failed to connect to the Postgres test database")
+	sqlDB, err := db.DB()
+	require.NoError(t, err, "Failed to get sql.DB")
+	t.Cleanup(func() {
+		require.NoError(t, sqlDB.Close(), "Failed to close the Postgres test database")
+	})
+
+	// Apply the embedded Postgres migrations
+	conn, err := sqlDB.Conn(t.Context())
+	require.NoError(t, err, "Failed to acquire migration connection")
+	defer conn.Close()
+	driver, err := postgresMigrate.WithConnection(t.Context(), conn, &postgresMigrate.Config{})
+	require.NoError(t, err, "Failed to create migration driver")
+	source, err := iofs.New(resources.FS, "migrations/postgres")
+	require.NoError(t, err, "Failed to create embedded migration source")
+	m, err := migrate.NewWithInstance("iofs", source, "pocket-id", driver)
+	require.NoError(t, err, "Failed to create migration instance")
+	require.NoError(t, m.Up(), "Failed to perform migrations")
+
+	return db
+}
+
 // openInMemoryTestDB opens a GORM instance backed by an in-memory SQLite database, unique to the test.
 func openInMemoryTestDB(t *testing.T) *gorm.DB {
 	t.Helper()