From 8ab3dd437b3c43371d542ef90f4ec32602f34c42 Mon Sep 17 00:00:00 2001 From: Elias Schneider Date: Tue, 6 Oct 2026 19:52:55 +0200 Subject: [PATCH] fix: export from SQLite fails to import into Postgres because of mismatched schemas --- .github/workflows/unit-tests.yml | 8 + AGENTS.md | 7 + backend/internal/service/export_service.go | 14 +- backend/internal/service/import_service.go | 9 +- .../internal/utils/db_schema_parity_test.go | 52 +++ backend/internal/utils/db_util.go | 32 ++ backend/internal/utils/testing/database.go | 61 +++ ..._sqlite_postgres_schema_alignment.down.sql | 7 + ...00_sqlite_postgres_schema_alignment.up.sql | 24 ++ ..._sqlite_postgres_schema_alignment.down.sql | 373 +++++++++++++++++ ...00_sqlite_postgres_schema_alignment.up.sql | 380 ++++++++++++++++++ 11 files changed, 956 insertions(+), 11 deletions(-) create mode 100644 backend/internal/utils/db_schema_parity_test.go create mode 100644 backend/resources/migrations/postgres/20261006120000_sqlite_postgres_schema_alignment.down.sql create mode 100644 backend/resources/migrations/postgres/20261006120000_sqlite_postgres_schema_alignment.up.sql create mode 100644 backend/resources/migrations/sqlite/20261006120000_sqlite_postgres_schema_alignment.down.sql create mode 100644 backend/resources/migrations/sqlite/20261006120000_sqlite_postgres_schema_alignment.up.sql diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 461abc92..38694909 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -34,6 +34,14 @@ jobs: working-directory: backend run: | go get ./... + - name: Start Postgres + # The SQLite/Postgres schema parity test needs Postgres and fails instead of skipping when POCKET_ID_TEST_POSTGRES_REQUIRED is set + if: runner.os == 'Linux' + run: | + docker run -d --name postgres -e POSTGRES_PASSWORD=postgres -p 5432:5432 --health-cmd "pg_isready -U postgres" --health-interval 1s --health-retries 60 postgres:17 + timeout 60 sh -c 'until [ "$(docker inspect -f "{{.State.Health.Status}}" postgres)" = healthy ]; do sleep 1; done' + echo "POCKET_ID_TEST_POSTGRES_URL=postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable" >> "$GITHUB_ENV" + echo "POCKET_ID_TEST_POSTGRES_REQUIRED=true" >> "$GITHUB_ENV" - name: Run backend unit tests working-directory: backend run: go test "-tags=exclude_frontend,unit" -v ./... diff --git a/AGENTS.md b/AGENTS.md index 8db356fa..20a3c397 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,6 +15,9 @@ the binary for production). This file lists what isn't obvious from reading the # backend/ — the exclude_frontend and unit tags are mandatory locally; CI uses them too go test -tags=exclude_frontend,unit ./... # unit/integration tests go test -tags=exclude_frontend,unit -run TestName ./internal/... # a single test +# Tests that need Postgres are skipped unless POCKET_ID_TEST_POSTGRES_URL points at a server (each test creates its own database) +docker run -d --rm --name pocket-id-test-pg -e POSTGRES_PASSWORD=postgres -p 55432:5432 postgres:17 +POCKET_ID_TEST_POSTGRES_URL='postgres://postgres:postgres@localhost:55432/postgres?sslmode=disable' go test -tags=exclude_frontend,unit ./... golangci-lint run # lint (config: backend/.golangci.yml - includes build tags) # frontend/ (or root) @@ -44,6 +47,10 @@ cd ../.. && pnpm test # = playwright test in tests/ timelines. Add a matching up/down pair to **both**. Not GORM AutoMigrate. SQLite migrations are not auto-wrapped in a transaction (`NoTxWrap`); wrap multi-statement ones manually (`PRAGMA foreign_keys=OFF; BEGIN; … COMMIT; PRAGMA foreign_keys=ON;`). +- **Column types must match across DBs for export/import.** A SQLite export must import into Postgres + and vice versa, so every column must map to the same `DBExportKind` (`backend/internal/utils/db_util.go`) + on both: JSON/binary is `JSONB`/`BYTEA` ↔ `BLOB`, timestamps are `TIMESTAMPTZ` ↔ `DATETIME` (never + `INTEGER`/`TEXT`), and nullability must match. `TestDBSchemaParity` enforces this (needs Postgres, see above). ## Backend (Go) diff --git a/backend/internal/service/export_service.go b/backend/internal/service/export_service.go index d6261c03..aa3e3510 100644 --- a/backend/internal/service/export_service.go +++ b/backend/internal/service/export_service.go @@ -160,38 +160,36 @@ func getScanValuesForTable(cols []string, types utils.DBSchemaTableTypes) []any // Store a pointer // Note: don't create a helper function for this switch, because it would return type "any" and mess everything up // If the column is nullable, we need a pointer to a pointer! - switch types[col].Name { - case "boolean", "bool": + switch types[col].ExportKind() { + case utils.DBExportKindBool: var x bool if types[col].Nullable { res[i] = new(new(x)) } else { res[i] = new(x) } - case "blob", "bytea", "jsonb": - // Treat jsonb columns as binary too + case utils.DBExportKindBytes: var x []byte if types[col].Nullable { res[i] = new(new(x)) } else { res[i] = new(x) } - case "timestamp", "timestamptz", "timestamp with time zone", "datetime": + case utils.DBExportKindDateTime: var x datatype.DateTime if types[col].Nullable { res[i] = new(new(x)) } else { res[i] = new(x) } - case "integer", "int", "bigint": + case utils.DBExportKindInteger: var x int64 if types[col].Nullable { res[i] = new(new(x)) } else { res[i] = new(x) } - default: - // Treat everything else as a string (including the "numeric" type) + case utils.DBExportKindString: var x string if types[col].Nullable { res[i] = new(new(x)) diff --git a/backend/internal/service/import_service.go b/backend/internal/service/import_service.go index 1946026c..e70fd3d6 100644 --- a/backend/internal/service/import_service.go +++ b/backend/internal/service/import_service.go @@ -367,8 +367,8 @@ func normalizeRowWithSchema(row map[string]any, table string, schema utils.DBSch colType := schema[table][col] - switch colType.Name { - case "timestamp", "timestamptz", "timestamp with time zone", "datetime": + switch colType.ExportKind() { + case utils.DBExportKindDateTime: // Dates are stored as strings str, ok := val.(string) if !ok { @@ -380,7 +380,7 @@ func normalizeRowWithSchema(row map[string]any, table string, schema utils.DBSch } row[col] = d - case "blob", "bytea", "jsonb": + case utils.DBExportKindBytes: // Binary data and jsonb data is stored in the file as base64-encoded string str, ok := val.(string) if !ok { @@ -397,6 +397,9 @@ func normalizeRowWithSchema(row map[string]any, table string, schema utils.DBSch } else { row[col] = b } + + case utils.DBExportKindString, utils.DBExportKindBool, utils.DBExportKindInteger: + // JSON decodes these values into types the database accepts as they are } } diff --git a/backend/internal/utils/db_schema_parity_test.go b/backend/internal/utils/db_schema_parity_test.go new file mode 100644 index 00000000..03d2cf36 --- /dev/null +++ b/backend/internal/utils/db_schema_parity_test.go @@ -0,0 +1,52 @@ +//go:build unit + +package utils_test + +import ( + "maps" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/pocket-id/pocket-id/backend/internal/utils" + testutils "github.com/pocket-id/pocket-id/backend/internal/utils/testing" +) + +// TestDBSchemaParity ensures the SQLite and Postgres migrations produce schemas that encode every column the same way in a data export +// Without this, an export taken from one database provider fails to import into the other (see https://github.com/pocket-id/pocket-id/issues/1603 and https://github.com/pocket-id/pocket-id/issues/1815) +// If this test fails after adding a migration, pick column types for both databases that map to the same utils.DBExportKind +func TestDBSchemaParity(t *testing.T) { + sqliteSchema, err := utils.LoadDBSchemaTypes(testutils.NewDatabaseForTest(t)) + require.NoError(t, err) + postgresSchema, err := utils.LoadDBSchemaTypes(testutils.NewPostgresDatabaseForTest(t)) + require.NoError(t, err) + + // The migration bookkeeping table is managed by golang-migrate and never exported + delete(sqliteSchema, "schema_migrations") + delete(postgresSchema, "schema_migrations") + + // Both databases must have the same tables + require.ElementsMatch(t, slices.Collect(maps.Keys(sqliteSchema)), slices.Collect(maps.Keys(postgresSchema)), "SQLite and Postgres must have the same tables") + + for table, sqliteColumns := range sqliteSchema { + postgresColumns := postgresSchema[table] + + // Both databases must have the same columns in each table + if !assert.ElementsMatchf(t, slices.Collect(maps.Keys(sqliteColumns)), slices.Collect(maps.Keys(postgresColumns)), "SQLite and Postgres must have the same columns in table %q", table) { + continue + } + + // Each column must be encoded the same way in an export and accept the same values + for column, sqliteColumn := range sqliteColumns { + postgresColumn := postgresColumns[column] + assert.Equalf(t, sqliteColumn.ExportKind(), postgresColumn.ExportKind(), + "column %s.%s is exported differently: SQLite type %q is %q, Postgres type %q is %q", + table, column, sqliteColumn.Name, sqliteColumn.ExportKind(), postgresColumn.Name, postgresColumn.ExportKind()) + assert.Equalf(t, sqliteColumn.Nullable, postgresColumn.Nullable, + "column %s.%s has a different nullability: SQLite nullable=%t, Postgres nullable=%t", + table, column, sqliteColumn.Nullable, postgresColumn.Nullable) + } + } +} diff --git a/backend/internal/utils/db_util.go b/backend/internal/utils/db_util.go index 6e7141ec..5895fb5e 100644 --- a/backend/internal/utils/db_util.go +++ b/backend/internal/utils/db_util.go @@ -37,6 +37,38 @@ type DBSchemaColumn struct { Name string Nullable bool } + +// DBExportKind is how the values of a column are encoded in a data export +type DBExportKind string + +const ( + DBExportKindString DBExportKind = "string" + DBExportKindBool DBExportKind = "bool" + DBExportKindInteger DBExportKind = "integer" + DBExportKindDateTime DBExportKind = "datetime" + // DBExportKindBytes values are stored as base64-encoded strings + DBExportKindBytes DBExportKind = "bytes" +) + +// ExportKind returns how the values of the column are encoded in a data export +// An export from one database provider can only be imported into another if every column has the same export kind on both +func (c DBSchemaColumn) ExportKind() DBExportKind { + switch c.Name { + case "boolean", "bool": + return DBExportKindBool + case "integer", "int", "bigint": + return DBExportKindInteger + case "timestamp", "timestamptz", "timestamp with time zone", "datetime": + return DBExportKindDateTime + case "blob", "bytea", "jsonb": + // jsonb is treated as binary too + return DBExportKindBytes + default: + // Everything else is treated as a string, including the "numeric" type + return DBExportKindString + } +} + type DBSchemaTableTypes = map[string]DBSchemaColumn type DBSchemaTypes = map[string]DBSchemaTableTypes diff --git a/backend/internal/utils/testing/database.go b/backend/internal/utils/testing/database.go index e0394a42..2c851bbd 100644 --- a/backend/internal/utils/testing/database.go +++ b/backend/internal/utils/testing/database.go @@ -5,20 +5,26 @@ package testing import ( + "crypto/rand" "errors" "log/slog" + "net/url" + "os" "path/filepath" + "strings" "testing" "time" _ "github.com/golang-migrate/migrate/v4/source/file" "github.com/golang-migrate/migrate/v4" + postgresMigrate "github.com/golang-migrate/migrate/v4/database/postgres" sqliteMigrate "github.com/golang-migrate/migrate/v4/database/sqlite3" "github.com/golang-migrate/migrate/v4/source/iofs" sqlitekit "github.com/italypaleale/go-sql-utils/sqlite" "github.com/libtnb/sqlite" "github.com/stretchr/testify/require" + "gorm.io/driver/postgres" "gorm.io/gorm" "gorm.io/gorm/logger" @@ -62,6 +68,61 @@ func NewConcurrentDatabaseForTest(t *testing.T) *gorm.DB { return db } +// NewPostgresDatabaseForTest returns a new instance of GORM connected to a fresh Postgres database with all Postgres migrations applied. +// The server is taken from the POCKET_ID_TEST_POSTGRES_URL environment variable, and a new database is created on it for each test and dropped afterwards. +// When the variable is not set the test is skipped, unless POCKET_ID_TEST_POSTGRES_REQUIRED is "true", which CI sets so these tests can never be skipped silently there. +func NewPostgresDatabaseForTest(t *testing.T) *gorm.DB { + t.Helper() + + serverURL := os.Getenv("POCKET_ID_TEST_POSTGRES_URL") + if serverURL == "" { + if os.Getenv("POCKET_ID_TEST_POSTGRES_REQUIRED") == "true" { + t.Fatal("POCKET_ID_TEST_POSTGRES_URL must be set when POCKET_ID_TEST_POSTGRES_REQUIRED is true") + } + t.Skip("POCKET_ID_TEST_POSTGRES_URL is not set") + } + + // Create a database that only this test uses, so tests can run in parallel against the same server + admin, err := gorm.Open(postgres.Open(serverURL), newTestGormConfig(t)) + require.NoError(t, err, "Failed to connect to the Postgres test server") + adminDB, err := admin.DB() + require.NoError(t, err, "Failed to get sql.DB") + dbName := "pocket_id_test_" + strings.ToLower(rand.Text()) + require.NoError(t, admin.Exec(`CREATE DATABASE "`+dbName+`"`).Error, "Failed to create the Postgres test database") + + // Drop the database once the test is done, after its own connections have been closed by the cleanup registered below + t.Cleanup(func() { + require.NoError(t, admin.Exec(`DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`).Error, "Failed to drop the Postgres test database") + require.NoError(t, adminDB.Close(), "Failed to close the Postgres test server connection") + }) + + // Connect to the new database by swapping the database name in the server URL + u, err := url.Parse(serverURL) + require.NoError(t, err, "Failed to parse POCKET_ID_TEST_POSTGRES_URL") + u.Path = "/" + dbName + db, err := gorm.Open(postgres.Open(u.String()), newTestGormConfig(t)) + require.NoError(t, err, "Failed to connect to the Postgres test database") + sqlDB, err := db.DB() + require.NoError(t, err, "Failed to get sql.DB") + t.Cleanup(func() { + require.NoError(t, sqlDB.Close(), "Failed to close the Postgres test database") + }) + + // Apply the embedded Postgres migrations + conn, err := sqlDB.Conn(t.Context()) + require.NoError(t, err, "Failed to acquire migration connection") + defer conn.Close() + driver, err := postgresMigrate.WithConnection(t.Context(), conn, &postgresMigrate.Config{}) + require.NoError(t, err, "Failed to create migration driver") + source, err := iofs.New(resources.FS, "migrations/postgres") + require.NoError(t, err, "Failed to create embedded migration source") + m, err := migrate.NewWithInstance("iofs", source, "pocket-id", driver) + require.NoError(t, err, "Failed to create migration instance") + require.NoError(t, m.Up(), "Failed to perform migrations") + + return db +} + // openInMemoryTestDB opens a GORM instance backed by an in-memory SQLite database, unique to the test. func openInMemoryTestDB(t *testing.T) *gorm.DB { t.Helper() diff --git a/backend/resources/migrations/postgres/20261006120000_sqlite_postgres_schema_alignment.down.sql b/backend/resources/migrations/postgres/20261006120000_sqlite_postgres_schema_alignment.down.sql new file mode 100644 index 00000000..f9a9bbc9 --- /dev/null +++ b/backend/resources/migrations/postgres/20261006120000_sqlite_postgres_schema_alignment.down.sql @@ -0,0 +1,7 @@ +ALTER TABLE audit_logs ALTER COLUMN created_at DROP NOT NULL; +ALTER TABLE audit_logs ALTER COLUMN user_agent DROP NOT NULL; +ALTER TABLE kv ALTER COLUMN value DROP NOT NULL; +ALTER TABLE oidc_clients ALTER COLUMN created_at DROP NOT NULL; +ALTER TABLE users ALTER COLUMN first_name DROP NOT NULL; +ALTER TABLE users ALTER COLUMN last_name DROP NOT NULL; +ALTER TABLE webauthn_credentials ALTER COLUMN created_at DROP NOT NULL; diff --git a/backend/resources/migrations/postgres/20261006120000_sqlite_postgres_schema_alignment.up.sql b/backend/resources/migrations/postgres/20261006120000_sqlite_postgres_schema_alignment.up.sql new file mode 100644 index 00000000..e7af62ed --- /dev/null +++ b/backend/resources/migrations/postgres/20261006120000_sqlite_postgres_schema_alignment.up.sql @@ -0,0 +1,24 @@ +-- Declare the columns that are NOT NULL on SQLite as NOT NULL too, so an export from either database can be imported into the other +-- The application never writes NULL into these columns, but backfill any NULL left over from older versions first +-- Empty strings are what the application writes for these values and what readers of kv treat as missing + +UPDATE audit_logs SET created_at = now() WHERE created_at IS NULL; +ALTER TABLE audit_logs ALTER COLUMN created_at SET NOT NULL; + +UPDATE audit_logs SET user_agent = '' WHERE user_agent IS NULL; +ALTER TABLE audit_logs ALTER COLUMN user_agent SET NOT NULL; + +UPDATE kv SET value = '' WHERE value IS NULL; +ALTER TABLE kv ALTER COLUMN value SET NOT NULL; + +UPDATE oidc_clients SET created_at = now() WHERE created_at IS NULL; +ALTER TABLE oidc_clients ALTER COLUMN created_at SET NOT NULL; + +UPDATE users SET first_name = '' WHERE first_name IS NULL; +ALTER TABLE users ALTER COLUMN first_name SET NOT NULL; + +UPDATE users SET last_name = '' WHERE last_name IS NULL; +ALTER TABLE users ALTER COLUMN last_name SET NOT NULL; + +UPDATE webauthn_credentials SET created_at = now() WHERE created_at IS NULL; +ALTER TABLE webauthn_credentials ALTER COLUMN created_at SET NOT NULL; diff --git a/backend/resources/migrations/sqlite/20261006120000_sqlite_postgres_schema_alignment.down.sql b/backend/resources/migrations/sqlite/20261006120000_sqlite_postgres_schema_alignment.down.sql new file mode 100644 index 00000000..4d427bdc --- /dev/null +++ b/backend/resources/migrations/sqlite/20261006120000_sqlite_postgres_schema_alignment.down.sql @@ -0,0 +1,373 @@ +PRAGMA foreign_keys = OFF; +BEGIN; + +-- Restore the previous column types of webauthn_sessions.extensions and oauth2_sessions.rotated_at +CREATE TABLE webauthn_sessions_new +( + id TEXT NOT NULL PRIMARY KEY, + created_at DATETIME, + challenge TEXT NOT NULL UNIQUE, + expires_at DATETIME NOT NULL, + user_verification TEXT NOT NULL, + credential_params BLOB DEFAULT '[]' NOT NULL, + extensions TEXT NOT NULL DEFAULT '{}' +); + +INSERT INTO webauthn_sessions_new ( + id, + created_at, + challenge, + expires_at, + user_verification, + credential_params, + extensions +) +SELECT + id, + created_at, + challenge, + expires_at, + user_verification, + credential_params, + extensions +FROM webauthn_sessions; + +DROP TABLE webauthn_sessions; +ALTER TABLE webauthn_sessions_new RENAME TO webauthn_sessions; + +CREATE INDEX idx_webauthn_sessions_expires_at ON webauthn_sessions (expires_at); + +CREATE TABLE oauth2_sessions_new ( + id TEXT NOT NULL PRIMARY KEY, + created_at DATETIME NOT NULL, + kind TEXT NOT NULL, + key TEXT NOT NULL, + request_id TEXT NOT NULL, + client_id TEXT NOT NULL REFERENCES oidc_clients(id) ON DELETE CASCADE, + access_token_signature TEXT NOT NULL DEFAULT '', + active BOOLEAN NOT NULL DEFAULT TRUE, + request_data BLOB NOT NULL, + expires_at DATETIME, + rotated_at INTEGER, + CONSTRAINT chk_oauth2_sessions_client_id + CHECK (client_id = json_extract(CAST(request_data AS TEXT), '$.client_id')) +); + +INSERT INTO oauth2_sessions_new ( + id, + created_at, + kind, + key, + request_id, + client_id, + access_token_signature, + active, + request_data, + expires_at, + rotated_at +) +SELECT + id, + created_at, + kind, + key, + request_id, + client_id, + access_token_signature, + active, + request_data, + expires_at, + rotated_at +FROM oauth2_sessions; + +DROP TABLE oauth2_sessions; +ALTER TABLE oauth2_sessions_new RENAME TO oauth2_sessions; + +CREATE UNIQUE INDEX idx_oauth2_sessions_kind_key ON oauth2_sessions (kind, key); +CREATE INDEX idx_oauth2_sessions_kind_request ON oauth2_sessions (kind, request_id); +CREATE INDEX idx_oauth2_sessions_expires_at ON oauth2_sessions (expires_at); +CREATE INDEX idx_oauth2_sessions_client_subject + ON oauth2_sessions (client_id, json_extract(CAST(request_data AS TEXT), '$.session.subject'), kind, active); + +-- Restore the TEXT primary keys without the NOT NULL constraint + +CREATE TABLE api_keys_new +( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + key TEXT NOT NULL UNIQUE, + description TEXT, + expires_at DATETIME NOT NULL, + last_used_at DATETIME, + created_at DATETIME, + user_id TEXT REFERENCES users(id) ON DELETE CASCADE, + expiration_email_sent BOOLEAN NOT NULL DEFAULT 0 +); + +INSERT INTO api_keys_new ( + id, + name, + key, + description, + expires_at, + last_used_at, + created_at, + user_id, + expiration_email_sent +) +SELECT + id, + name, + key, + description, + expires_at, + last_used_at, + created_at, + user_id, + expiration_email_sent +FROM api_keys; + +DROP TABLE api_keys; +ALTER TABLE api_keys_new RENAME TO api_keys; + +CREATE INDEX idx_api_keys_key ON api_keys(key); +CREATE INDEX idx_api_keys_expires_at ON api_keys(expires_at); + +CREATE TABLE audit_logs_new +( + id TEXT PRIMARY KEY, + created_at DATETIME NOT NULL, + event TEXT NOT NULL, + ip_address TEXT, + user_agent TEXT NOT NULL, + data BLOB NOT NULL, + user_id TEXT REFERENCES users ON DELETE CASCADE, + country TEXT, + city TEXT +); + +INSERT INTO audit_logs_new ( + id, + created_at, + event, + ip_address, + user_agent, + data, + user_id, + country, + city +) +SELECT + id, + created_at, + event, + ip_address, + user_agent, + data, + user_id, + country, + city +FROM audit_logs; + +DROP TABLE audit_logs; +ALTER TABLE audit_logs_new RENAME TO audit_logs; + +CREATE INDEX idx_audit_logs_client_name ON audit_logs((json_extract(data, '$.clientName'))); +CREATE INDEX idx_audit_logs_country ON audit_logs (country); +CREATE INDEX idx_audit_logs_created_at ON audit_logs (created_at); +CREATE INDEX idx_audit_logs_event ON audit_logs (event); +CREATE INDEX idx_audit_logs_user_agent ON audit_logs (user_agent); +CREATE INDEX idx_audit_logs_user_id ON audit_logs (user_id); + +CREATE TABLE oidc_clients_new +( + id TEXT PRIMARY KEY, + created_at DATETIME NOT NULL, + name TEXT, + callback_urls BLOB, + image_type TEXT, + created_by_id TEXT REFERENCES users ON DELETE SET NULL, + is_public BOOLEAN DEFAULT FALSE, + pkce_enabled BOOLEAN DEFAULT FALSE, + logout_callback_urls BLOB, + credentials BLOB, + launch_url TEXT, + requires_reauthentication BOOLEAN NOT NULL DEFAULT FALSE, + dark_image_type TEXT, + is_group_restricted BOOLEAN NOT NULL DEFAULT 0, + requires_pushed_authorization_requests BOOLEAN NOT NULL DEFAULT FALSE, + skip_consent BOOLEAN NOT NULL DEFAULT FALSE, + pkce_supported BOOLEAN NOT NULL DEFAULT 0, + description TEXT NOT NULL DEFAULT '', + client_type TEXT NOT NULL DEFAULT 'standard', + metadata_expires_at DATETIME, + metadata_grant_types BLOB, + access_token_duration_minutes INTEGER NOT NULL DEFAULT 60, + refresh_token_duration_minutes INTEGER NOT NULL DEFAULT 43200, + backchannel_logout_url TEXT NOT NULL DEFAULT '' +); + +INSERT INTO oidc_clients_new ( + id, + created_at, + name, + callback_urls, + image_type, + created_by_id, + is_public, + pkce_enabled, + logout_callback_urls, + credentials, + launch_url, + requires_reauthentication, + dark_image_type, + is_group_restricted, + requires_pushed_authorization_requests, + skip_consent, + pkce_supported, + description, + client_type, + metadata_expires_at, + metadata_grant_types, + access_token_duration_minutes, + refresh_token_duration_minutes, + backchannel_logout_url +) +SELECT + id, + created_at, + name, + callback_urls, + image_type, + created_by_id, + is_public, + pkce_enabled, + logout_callback_urls, + credentials, + launch_url, + requires_reauthentication, + dark_image_type, + is_group_restricted, + requires_pushed_authorization_requests, + skip_consent, + pkce_supported, + description, + client_type, + metadata_expires_at, + metadata_grant_types, + access_token_duration_minutes, + refresh_token_duration_minutes, + backchannel_logout_url +FROM oidc_clients; + +DROP TABLE oidc_clients; +ALTER TABLE oidc_clients_new RENAME TO oidc_clients; + +CREATE TABLE reauthentication_tokens_new +( + id TEXT PRIMARY KEY, + created_at DATETIME NOT NULL, + token TEXT NOT NULL UNIQUE, + expires_at DATETIME NOT NULL, + user_id TEXT NOT NULL REFERENCES users ON DELETE CASCADE +); + +INSERT INTO reauthentication_tokens_new ( + id, + created_at, + token, + expires_at, + user_id +) +SELECT + id, + created_at, + token, + expires_at, + user_id +FROM reauthentication_tokens; + +DROP TABLE reauthentication_tokens; +ALTER TABLE reauthentication_tokens_new RENAME TO reauthentication_tokens; + +CREATE INDEX idx_reauthentication_tokens_token ON reauthentication_tokens (token); +CREATE INDEX idx_reauthentication_tokens_expires_at ON reauthentication_tokens (expires_at); + +CREATE TABLE scim_service_providers_new +( + id TEXT PRIMARY KEY, + created_at DATETIME NOT NULL, + endpoint TEXT NOT NULL, + token TEXT NOT NULL, + last_synced_at DATETIME, + oidc_client_id TEXT NOT NULL, + FOREIGN KEY (oidc_client_id) REFERENCES oidc_clients (id) ON DELETE CASCADE +); + +INSERT INTO scim_service_providers_new ( + id, + created_at, + endpoint, + token, + last_synced_at, + oidc_client_id +) +SELECT + id, + created_at, + endpoint, + token, + last_synced_at, + oidc_client_id +FROM scim_service_providers; + +DROP TABLE scim_service_providers; +ALTER TABLE scim_service_providers_new RENAME TO scim_service_providers; + +CREATE TABLE webauthn_credentials_new +( + id TEXT PRIMARY KEY, + created_at DATETIME NOT NULL, + name TEXT NOT NULL, + credential_id BLOB NOT NULL UNIQUE, + public_key BLOB NOT NULL, + attestation_type TEXT NOT NULL, + transport BLOB NOT NULL, + user_id TEXT REFERENCES users ON DELETE CASCADE, + backup_eligible BOOLEAN DEFAULT FALSE NOT NULL, + backup_state BOOLEAN DEFAULT FALSE NOT NULL, + aaguid TEXT NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000' +); + +INSERT INTO webauthn_credentials_new ( + id, + created_at, + name, + credential_id, + public_key, + attestation_type, + transport, + user_id, + backup_eligible, + backup_state, + aaguid +) +SELECT + id, + created_at, + name, + credential_id, + public_key, + attestation_type, + transport, + user_id, + backup_eligible, + backup_state, + aaguid +FROM webauthn_credentials; + +DROP TABLE webauthn_credentials; +ALTER TABLE webauthn_credentials_new RENAME TO webauthn_credentials; + +COMMIT; +PRAGMA foreign_keys = ON; diff --git a/backend/resources/migrations/sqlite/20261006120000_sqlite_postgres_schema_alignment.up.sql b/backend/resources/migrations/sqlite/20261006120000_sqlite_postgres_schema_alignment.up.sql new file mode 100644 index 00000000..2062903c --- /dev/null +++ b/backend/resources/migrations/sqlite/20261006120000_sqlite_postgres_schema_alignment.up.sql @@ -0,0 +1,380 @@ +PRAGMA foreign_keys = OFF; +BEGIN; + +-- Align the webauthn_sessions.extensions (JSONB) and oauth2_sessions.rotated_at (TIMESTAMPTZ) column types with the export format used for PostgreSQL +CREATE TABLE webauthn_sessions_new +( + id TEXT NOT NULL PRIMARY KEY, + created_at DATETIME, + challenge TEXT NOT NULL UNIQUE, + expires_at DATETIME NOT NULL, + user_verification TEXT NOT NULL, + credential_params BLOB DEFAULT '[]' NOT NULL, + extensions BLOB NOT NULL DEFAULT '{}' +); + +INSERT INTO webauthn_sessions_new ( + id, + created_at, + challenge, + expires_at, + user_verification, + credential_params, + extensions +) +SELECT + id, + created_at, + challenge, + expires_at, + user_verification, + credential_params, + extensions +FROM webauthn_sessions; + +DROP TABLE webauthn_sessions; +ALTER TABLE webauthn_sessions_new RENAME TO webauthn_sessions; + +CREATE INDEX idx_webauthn_sessions_expires_at ON webauthn_sessions (expires_at); + +CREATE TABLE oauth2_sessions_new ( + id TEXT NOT NULL PRIMARY KEY, + created_at DATETIME NOT NULL, + kind TEXT NOT NULL, + key TEXT NOT NULL, + request_id TEXT NOT NULL, + client_id TEXT NOT NULL REFERENCES oidc_clients(id) ON DELETE CASCADE, + access_token_signature TEXT NOT NULL DEFAULT '', + active BOOLEAN NOT NULL DEFAULT TRUE, + request_data BLOB NOT NULL, + expires_at DATETIME, + rotated_at DATETIME, + CONSTRAINT chk_oauth2_sessions_client_id + CHECK (client_id = json_extract(CAST(request_data AS TEXT), '$.client_id')) +); + +INSERT INTO oauth2_sessions_new ( + id, + created_at, + kind, + key, + request_id, + client_id, + access_token_signature, + active, + request_data, + expires_at, + rotated_at +) +SELECT + id, + created_at, + kind, + key, + request_id, + client_id, + access_token_signature, + active, + request_data, + expires_at, + rotated_at +FROM oauth2_sessions; + +DROP TABLE oauth2_sessions; +ALTER TABLE oauth2_sessions_new RENAME TO oauth2_sessions; + +CREATE UNIQUE INDEX idx_oauth2_sessions_kind_key ON oauth2_sessions (kind, key); +CREATE INDEX idx_oauth2_sessions_kind_request ON oauth2_sessions (kind, request_id); +CREATE INDEX idx_oauth2_sessions_expires_at ON oauth2_sessions (expires_at); +CREATE INDEX idx_oauth2_sessions_client_subject + ON oauth2_sessions (client_id, json_extract(CAST(request_data AS TEXT), '$.session.subject'), kind, active); + +-- Declare the TEXT primary keys as NOT NULL, because SQLite (unlike PostgreSQL) otherwise accepts NULL in them +-- Rows with a NULL id cannot be referenced or addressed by the application, so they are dropped + +CREATE TABLE api_keys_new +( + id TEXT NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + key TEXT NOT NULL UNIQUE, + description TEXT, + expires_at DATETIME NOT NULL, + last_used_at DATETIME, + created_at DATETIME, + user_id TEXT REFERENCES users(id) ON DELETE CASCADE, + expiration_email_sent BOOLEAN NOT NULL DEFAULT 0 +); + +INSERT INTO api_keys_new ( + id, + name, + key, + description, + expires_at, + last_used_at, + created_at, + user_id, + expiration_email_sent +) +SELECT + id, + name, + key, + description, + expires_at, + last_used_at, + created_at, + user_id, + expiration_email_sent +FROM api_keys +WHERE id IS NOT NULL; + +DROP TABLE api_keys; +ALTER TABLE api_keys_new RENAME TO api_keys; + +CREATE INDEX idx_api_keys_key ON api_keys(key); +CREATE INDEX idx_api_keys_expires_at ON api_keys(expires_at); + +CREATE TABLE audit_logs_new +( + id TEXT NOT NULL PRIMARY KEY, + created_at DATETIME NOT NULL, + event TEXT NOT NULL, + ip_address TEXT, + user_agent TEXT NOT NULL, + data BLOB NOT NULL, + user_id TEXT REFERENCES users ON DELETE CASCADE, + country TEXT, + city TEXT +); + +INSERT INTO audit_logs_new ( + id, + created_at, + event, + ip_address, + user_agent, + data, + user_id, + country, + city +) +SELECT + id, + created_at, + event, + ip_address, + user_agent, + data, + user_id, + country, + city +FROM audit_logs +WHERE id IS NOT NULL; + +DROP TABLE audit_logs; +ALTER TABLE audit_logs_new RENAME TO audit_logs; + +CREATE INDEX idx_audit_logs_client_name ON audit_logs((json_extract(data, '$.clientName'))); +CREATE INDEX idx_audit_logs_country ON audit_logs (country); +CREATE INDEX idx_audit_logs_created_at ON audit_logs (created_at); +CREATE INDEX idx_audit_logs_event ON audit_logs (event); +CREATE INDEX idx_audit_logs_user_agent ON audit_logs (user_agent); +CREATE INDEX idx_audit_logs_user_id ON audit_logs (user_id); + +CREATE TABLE oidc_clients_new +( + id TEXT NOT NULL PRIMARY KEY, + created_at DATETIME NOT NULL, + name TEXT, + callback_urls BLOB, + image_type TEXT, + created_by_id TEXT REFERENCES users ON DELETE SET NULL, + is_public BOOLEAN DEFAULT FALSE, + pkce_enabled BOOLEAN DEFAULT FALSE, + logout_callback_urls BLOB, + credentials BLOB, + launch_url TEXT, + requires_reauthentication BOOLEAN NOT NULL DEFAULT FALSE, + dark_image_type TEXT, + is_group_restricted BOOLEAN NOT NULL DEFAULT 0, + requires_pushed_authorization_requests BOOLEAN NOT NULL DEFAULT FALSE, + skip_consent BOOLEAN NOT NULL DEFAULT FALSE, + pkce_supported BOOLEAN NOT NULL DEFAULT 0, + description TEXT NOT NULL DEFAULT '', + client_type TEXT NOT NULL DEFAULT 'standard', + metadata_expires_at DATETIME, + metadata_grant_types BLOB, + access_token_duration_minutes INTEGER NOT NULL DEFAULT 60, + refresh_token_duration_minutes INTEGER NOT NULL DEFAULT 43200, + backchannel_logout_url TEXT NOT NULL DEFAULT '' +); + +INSERT INTO oidc_clients_new ( + id, + created_at, + name, + callback_urls, + image_type, + created_by_id, + is_public, + pkce_enabled, + logout_callback_urls, + credentials, + launch_url, + requires_reauthentication, + dark_image_type, + is_group_restricted, + requires_pushed_authorization_requests, + skip_consent, + pkce_supported, + description, + client_type, + metadata_expires_at, + metadata_grant_types, + access_token_duration_minutes, + refresh_token_duration_minutes, + backchannel_logout_url +) +SELECT + id, + created_at, + name, + callback_urls, + image_type, + created_by_id, + is_public, + pkce_enabled, + logout_callback_urls, + credentials, + launch_url, + requires_reauthentication, + dark_image_type, + is_group_restricted, + requires_pushed_authorization_requests, + skip_consent, + pkce_supported, + description, + client_type, + metadata_expires_at, + metadata_grant_types, + access_token_duration_minutes, + refresh_token_duration_minutes, + backchannel_logout_url +FROM oidc_clients +WHERE id IS NOT NULL; + +DROP TABLE oidc_clients; +ALTER TABLE oidc_clients_new RENAME TO oidc_clients; + +CREATE TABLE reauthentication_tokens_new +( + id TEXT NOT NULL PRIMARY KEY, + created_at DATETIME NOT NULL, + token TEXT NOT NULL UNIQUE, + expires_at DATETIME NOT NULL, + user_id TEXT NOT NULL REFERENCES users ON DELETE CASCADE +); + +INSERT INTO reauthentication_tokens_new ( + id, + created_at, + token, + expires_at, + user_id +) +SELECT + id, + created_at, + token, + expires_at, + user_id +FROM reauthentication_tokens +WHERE id IS NOT NULL; + +DROP TABLE reauthentication_tokens; +ALTER TABLE reauthentication_tokens_new RENAME TO reauthentication_tokens; + +CREATE INDEX idx_reauthentication_tokens_token ON reauthentication_tokens (token); +CREATE INDEX idx_reauthentication_tokens_expires_at ON reauthentication_tokens (expires_at); + +CREATE TABLE scim_service_providers_new +( + id TEXT NOT NULL PRIMARY KEY, + created_at DATETIME NOT NULL, + endpoint TEXT NOT NULL, + token TEXT NOT NULL, + last_synced_at DATETIME, + oidc_client_id TEXT NOT NULL, + FOREIGN KEY (oidc_client_id) REFERENCES oidc_clients (id) ON DELETE CASCADE +); + +INSERT INTO scim_service_providers_new ( + id, + created_at, + endpoint, + token, + last_synced_at, + oidc_client_id +) +SELECT + id, + created_at, + endpoint, + token, + last_synced_at, + oidc_client_id +FROM scim_service_providers +WHERE id IS NOT NULL; + +DROP TABLE scim_service_providers; +ALTER TABLE scim_service_providers_new RENAME TO scim_service_providers; + +CREATE TABLE webauthn_credentials_new +( + id TEXT NOT NULL PRIMARY KEY, + created_at DATETIME NOT NULL, + name TEXT NOT NULL, + credential_id BLOB NOT NULL UNIQUE, + public_key BLOB NOT NULL, + attestation_type TEXT NOT NULL, + transport BLOB NOT NULL, + user_id TEXT REFERENCES users ON DELETE CASCADE, + backup_eligible BOOLEAN DEFAULT FALSE NOT NULL, + backup_state BOOLEAN DEFAULT FALSE NOT NULL, + aaguid TEXT NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000' +); + +INSERT INTO webauthn_credentials_new ( + id, + created_at, + name, + credential_id, + public_key, + attestation_type, + transport, + user_id, + backup_eligible, + backup_state, + aaguid +) +SELECT + id, + created_at, + name, + credential_id, + public_key, + attestation_type, + transport, + user_id, + backup_eligible, + backup_state, + aaguid +FROM webauthn_credentials +WHERE id IS NOT NULL; + +DROP TABLE webauthn_credentials; +ALTER TABLE webauthn_credentials_new RENAME TO webauthn_credentials; + +COMMIT; +PRAGMA foreign_keys = ON;