sendnrw/pocket-id
2
0
Fork 0
mirror of https://github.com/pocket-id/pocket-id.git synced 2026-10-07 18:29:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: export from SQLite fails to import into Postgres because of mismatched schemas

Browse Source
This commit is contained in:
Elias Schneider
2026-10-06 20:00:22 +02:00
parent 69065e7cf0
commit 8ab3dd437b
11 changed files with 956 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/unit-tests.yml
+8
View File
@@ -34,6 +34,14 @@ jobs:
working-directory: backend
run: |
go get ./...
- name: Start Postgres
# The SQLite/Postgres schema parity test needs Postgres and fails instead of skipping when POCKET_ID_TEST_POSTGRES_REQUIRED is set
if: runner.os == 'Linux'
run: |
docker run -d --name postgres -e POSTGRES_PASSWORD=postgres -p 5432:5432 --health-cmd "pg_isready -U postgres" --health-interval 1s --health-retries 60 postgres:17
timeout 60 sh -c 'until [ "$(docker inspect -f "{{.State.Health.Status}}" postgres)" = healthy ]; do sleep 1; done'
echo "POCKET_ID_TEST_POSTGRES_URL=postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable" >> "$GITHUB_ENV"
echo "POCKET_ID_TEST_POSTGRES_REQUIRED=true" >> "$GITHUB_ENV"
- name: Run backend unit tests
working-directory: backend
run: go test "-tags=exclude_frontend,unit" -v ./...
AGENTS.md
+7
View File
@@ -15,6 +15,9 @@ the binary for production). This file lists what isn't obvious from reading the
# backend/ — the exclude_frontend and unit tags are mandatory locally; CI uses them too
go test -tags=exclude_frontend,unit ./... # unit/integration tests
go test -tags=exclude_frontend,unit -run TestName ./internal/... # a single test
# Tests that need Postgres are skipped unless POCKET_ID_TEST_POSTGRES_URL points at a server (each test creates its own database)
docker run -d --rm --name pocket-id-test-pg -e POSTGRES_PASSWORD=postgres -p 55432:5432 postgres:17
POCKET_ID_TEST_POSTGRES_URL='postgres://postgres:postgres@localhost:55432/postgres?sslmode=disable' go test -tags=exclude_frontend,unit ./...
golangci-lint run # lint (config: backend/.golangci.yml - includes build tags)
# frontend/ (or root)
@@ -44,6 +47,10 @@ cd ../.. && pnpm test # = playwright test in tests/
timelines. Add a matching up/down pair to **both**. Not GORM AutoMigrate. SQLite migrations
are not auto-wrapped in a transaction (`NoTxWrap`); wrap multi-statement ones manually
(`PRAGMA foreign_keys=OFF; BEGIN; … COMMIT; PRAGMA foreign_keys=ON;`).
- **Column types must match across DBs for export/import.** A SQLite export must import into Postgres
and vice versa, so every column must map to the same `DBExportKind` (`backend/internal/utils/db_util.go`)
on both: JSON/binary is `JSONB`/`BYTEA` ↔ `BLOB`, timestamps are `TIMESTAMPTZ` ↔ `DATETIME` (never
`INTEGER`/`TEXT`), and nullability must match. `TestDBSchemaParity` enforces this (needs Postgres, see above).
## Backend (Go)
backend/internal/service/export_service.go
+6 -8
View File
@@ -160,38 +160,36 @@ func getScanValuesForTable(cols []string, types utils.DBSchemaTableTypes) []any
// Store a pointer
// Note: don't create a helper function for this switch, because it would return type "any" and mess everything up
// If the column is nullable, we need a pointer to a pointer!
switch types[col].Name {
case "boolean", "bool":
switch types[col].ExportKind() {
case utils.DBExportKindBool:
var x bool
if types[col].Nullable {
res[i] = new(new(x))
} else {
res[i] = new(x)
}
case "blob", "bytea", "jsonb":
// Treat jsonb columns as binary too
case utils.DBExportKindBytes:
var x []byte
if types[col].Nullable {
res[i] = new(new(x))
} else {
res[i] = new(x)
}
case "timestamp", "timestamptz", "timestamp with time zone", "datetime":
case utils.DBExportKindDateTime:
var x datatype.DateTime
if types[col].Nullable {
res[i] = new(new(x))
} else {
res[i] = new(x)
}
case "integer", "int", "bigint":
case utils.DBExportKindInteger:
var x int64
if types[col].Nullable {
res[i] = new(new(x))
} else {
res[i] = new(x)
}
default:
// Treat everything else as a string (including the "numeric" type)
case utils.DBExportKindString:
var x string
if types[col].Nullable {
res[i] = new(new(x))
backend/internal/service/import_service.go
+6 -3
View File
@@ -367,8 +367,8 @@ func normalizeRowWithSchema(row map[string]any, table string, schema utils.DBSch
colType := schema[table][col]
switch colType.Name {
case "timestamp", "timestamptz", "timestamp with time zone", "datetime":
switch colType.ExportKind() {
case utils.DBExportKindDateTime:
// Dates are stored as strings
str, ok := val.(string)
if !ok {
@@ -380,7 +380,7 @@ func normalizeRowWithSchema(row map[string]any, table string, schema utils.DBSch
}
row[col] = d
case "blob", "bytea", "jsonb":
case utils.DBExportKindBytes:
// Binary data and jsonb data is stored in the file as base64-encoded string
str, ok := val.(string)
if !ok {
@@ -397,6 +397,9 @@ func normalizeRowWithSchema(row map[string]any, table string, schema utils.DBSch
} else {
row[col] = b
}
case utils.DBExportKindString, utils.DBExportKindBool, utils.DBExportKindInteger:
// JSON decodes these values into types the database accepts as they are
}
}
backend/internal/utils/db_schema_parity_test.go
+52
View File
@@ -0,0 +1,52 @@
//go:build unit
package utils_test
import (
"maps"
"slices"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/pocket-id/pocket-id/backend/internal/utils"
testutils "github.com/pocket-id/pocket-id/backend/internal/utils/testing"
)
// TestDBSchemaParity ensures the SQLite and Postgres migrations produce schemas that encode every column the same way in a data export
// Without this, an export taken from one database provider fails to import into the other (see https://github.com/pocket-id/pocket-id/issues/1603 and https://github.com/pocket-id/pocket-id/issues/1815)
// If this test fails after adding a migration, pick column types for both databases that map to the same utils.DBExportKind
func TestDBSchemaParity(t *testing.T) {
sqliteSchema, err := utils.LoadDBSchemaTypes(testutils.NewDatabaseForTest(t))
require.NoError(t, err)
postgresSchema, err := utils.LoadDBSchemaTypes(testutils.NewPostgresDatabaseForTest(t))
require.NoError(t, err)
// The migration bookkeeping table is managed by golang-migrate and never exported
delete(sqliteSchema, "schema_migrations")
delete(postgresSchema, "schema_migrations")
// Both databases must have the same tables
require.ElementsMatch(t, slices.Collect(maps.Keys(sqliteSchema)), slices.Collect(maps.Keys(postgresSchema)), "SQLite and Postgres must have the same tables")
for table, sqliteColumns := range sqliteSchema {
postgresColumns := postgresSchema[table]
// Both databases must have the same columns in each table
if !assert.ElementsMatchf(t, slices.Collect(maps.Keys(sqliteColumns)), slices.Collect(maps.Keys(postgresColumns)), "SQLite and Postgres must have the same columns in table %q", table) {
continue
}
// Each column must be encoded the same way in an export and accept the same values
for column, sqliteColumn := range sqliteColumns {
postgresColumn := postgresColumns[column]
assert.Equalf(t, sqliteColumn.ExportKind(), postgresColumn.ExportKind(),
"column %s.%s is exported differently: SQLite type %q is %q, Postgres type %q is %q",
table, column, sqliteColumn.Name, sqliteColumn.ExportKind(), postgresColumn.Name, postgresColumn.ExportKind())
assert.Equalf(t, sqliteColumn.Nullable, postgresColumn.Nullable,
"column %s.%s has a different nullability: SQLite nullable=%t, Postgres nullable=%t",
table, column, sqliteColumn.Nullable, postgresColumn.Nullable)
}
}
}
backend/internal/utils/db_util.go
+32
View File
@@ -37,6 +37,38 @@ type DBSchemaColumn struct {
Name string
Nullable bool
}
// DBExportKind is how the values of a column are encoded in a data export
type DBExportKind string
const (
DBExportKindString DBExportKind = "string"
DBExportKindBool DBExportKind = "bool"
DBExportKindInteger DBExportKind = "integer"
DBExportKindDateTime DBExportKind = "datetime"
// DBExportKindBytes values are stored as base64-encoded strings
DBExportKindBytes DBExportKind = "bytes"
)
// ExportKind returns how the values of the column are encoded in a data export
// An export from one database provider can only be imported into another if every column has the same export kind on both
func (c DBSchemaColumn) ExportKind() DBExportKind {
switch c.Name {
case "boolean", "bool":
return DBExportKindBool
case "integer", "int", "bigint":
return DBExportKindInteger
case "timestamp", "timestamptz", "timestamp with time zone", "datetime":
return DBExportKindDateTime
case "blob", "bytea", "jsonb":
// jsonb is treated as binary too
return DBExportKindBytes
default:
// Everything else is treated as a string, including the "numeric" type
return DBExportKindString
}
}
type DBSchemaTableTypes = map[string]DBSchemaColumn
type DBSchemaTypes = map[string]DBSchemaTableTypes
backend/internal/utils/testing/database.go
+61
View File
@@ -5,20 +5,26 @@
package testing
import (
"crypto/rand"
"errors"
"log/slog"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
"time"
_ "github.com/golang-migrate/migrate/v4/source/file"
"github.com/golang-migrate/migrate/v4"
postgresMigrate "github.com/golang-migrate/migrate/v4/database/postgres"
sqliteMigrate "github.com/golang-migrate/migrate/v4/database/sqlite3"
"github.com/golang-migrate/migrate/v4/source/iofs"
sqlitekit "github.com/italypaleale/go-sql-utils/sqlite"
"github.com/libtnb/sqlite"
"github.com/stretchr/testify/require"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"gorm.io/gorm/logger"
@@ -62,6 +68,61 @@ func NewConcurrentDatabaseForTest(t *testing.T) *gorm.DB {
return db
}
// NewPostgresDatabaseForTest returns a new instance of GORM connected to a fresh Postgres database with all Postgres migrations applied.
// The server is taken from the POCKET_ID_TEST_POSTGRES_URL environment variable, and a new database is created on it for each test and dropped afterwards.
// When the variable is not set the test is skipped, unless POCKET_ID_TEST_POSTGRES_REQUIRED is "true", which CI sets so these tests can never be skipped silently there.
func NewPostgresDatabaseForTest(t *testing.T) *gorm.DB {
t.Helper()
serverURL := os.Getenv("POCKET_ID_TEST_POSTGRES_URL")
if serverURL == "" {
if os.Getenv("POCKET_ID_TEST_POSTGRES_REQUIRED") == "true" {
t.Fatal("POCKET_ID_TEST_POSTGRES_URL must be set when POCKET_ID_TEST_POSTGRES_REQUIRED is true")
}
t.Skip("POCKET_ID_TEST_POSTGRES_URL is not set")
}
// Create a database that only this test uses, so tests can run in parallel against the same server
admin, err := gorm.Open(postgres.Open(serverURL), newTestGormConfig(t))
require.NoError(t, err, "Failed to connect to the Postgres test server")
adminDB, err := admin.DB()
require.NoError(t, err, "Failed to get sql.DB")
dbName := "pocket_id_test_" + strings.ToLower(rand.Text())
require.NoError(t, admin.Exec(`CREATE DATABASE "`+dbName+`"`).Error, "Failed to create the Postgres test database")
// Drop the database once the test is done, after its own connections have been closed by the cleanup registered below
t.Cleanup(func() {
require.NoError(t, admin.Exec(`DROP DATABASE IF EXISTS "`+dbName+`" WITH (FORCE)`).Error, "Failed to drop the Postgres test database")
require.NoError(t, adminDB.Close(), "Failed to close the Postgres test server connection")
})
// Connect to the new database by swapping the database name in the server URL
u, err := url.Parse(serverURL)
require.NoError(t, err, "Failed to parse POCKET_ID_TEST_POSTGRES_URL")
u.Path = "/" + dbName
db, err := gorm.Open(postgres.Open(u.String()), newTestGormConfig(t))
require.NoError(t, err, "Failed to connect to the Postgres test database")
sqlDB, err := db.DB()
require.NoError(t, err, "Failed to get sql.DB")
t.Cleanup(func() {
require.NoError(t, sqlDB.Close(), "Failed to close the Postgres test database")
})
// Apply the embedded Postgres migrations
conn, err := sqlDB.Conn(t.Context())
require.NoError(t, err, "Failed to acquire migration connection")
defer conn.Close()
driver, err := postgresMigrate.WithConnection(t.Context(), conn, &postgresMigrate.Config{})
require.NoError(t, err, "Failed to create migration driver")
source, err := iofs.New(resources.FS, "migrations/postgres")
require.NoError(t, err, "Failed to create embedded migration source")
m, err := migrate.NewWithInstance("iofs", source, "pocket-id", driver)
require.NoError(t, err, "Failed to create migration instance")
require.NoError(t, m.Up(), "Failed to perform migrations")
return db
}
// openInMemoryTestDB opens a GORM instance backed by an in-memory SQLite database, unique to the test.
func openInMemoryTestDB(t *testing.T) *gorm.DB {
t.Helper()
backend/resources/migrations/postgres/20261006120000_sqlite_postgres_schema_alignment.down.sql
+7
View File
@@ -0,0 +1,7 @@
ALTER TABLE audit_logs ALTER COLUMN created_at DROP NOT NULL;
ALTER TABLE audit_logs ALTER COLUMN user_agent DROP NOT NULL;
ALTER TABLE kv ALTER COLUMN value DROP NOT NULL;
ALTER TABLE oidc_clients ALTER COLUMN created_at DROP NOT NULL;
ALTER TABLE users ALTER COLUMN first_name DROP NOT NULL;
ALTER TABLE users ALTER COLUMN last_name DROP NOT NULL;
ALTER TABLE webauthn_credentials ALTER COLUMN created_at DROP NOT NULL;
backend/resources/migrations/postgres/20261006120000_sqlite_postgres_schema_alignment.up.sql
+24
View File
@@ -0,0 +1,24 @@
-- Declare the columns that are NOT NULL on SQLite as NOT NULL too, so an export from either database can be imported into the other
-- The application never writes NULL into these columns, but backfill any NULL left over from older versions first
-- Empty strings are what the application writes for these values and what readers of kv treat as missing
UPDATE audit_logs SET created_at = now() WHERE created_at IS NULL;
ALTER TABLE audit_logs ALTER COLUMN created_at SET NOT NULL;
UPDATE audit_logs SET user_agent = '' WHERE user_agent IS NULL;
ALTER TABLE audit_logs ALTER COLUMN user_agent SET NOT NULL;
UPDATE kv SET value = '' WHERE value IS NULL;
ALTER TABLE kv ALTER COLUMN value SET NOT NULL;
UPDATE oidc_clients SET created_at = now() WHERE created_at IS NULL;
ALTER TABLE oidc_clients ALTER COLUMN created_at SET NOT NULL;
UPDATE users SET first_name = '' WHERE first_name IS NULL;
ALTER TABLE users ALTER COLUMN first_name SET NOT NULL;
UPDATE users SET last_name = '' WHERE last_name IS NULL;
ALTER TABLE users ALTER COLUMN last_name SET NOT NULL;
UPDATE webauthn_credentials SET created_at = now() WHERE created_at IS NULL;
ALTER TABLE webauthn_credentials ALTER COLUMN created_at SET NOT NULL;
backend/resources/migrations/sqlite/20261006120000_sqlite_postgres_schema_alignment.down.sql
+373
View File
@@ -0,0 +1,373 @@
PRAGMA foreign_keys = OFF;
BEGIN;
-- Restore the previous column types of webauthn_sessions.extensions and oauth2_sessions.rotated_at
CREATE TABLE webauthn_sessions_new
(
id TEXT NOT NULL PRIMARY KEY,
created_at DATETIME,
challenge TEXT NOT NULL UNIQUE,
expires_at DATETIME NOT NULL,
user_verification TEXT NOT NULL,
credential_params BLOB DEFAULT '[]' NOT NULL,
extensions TEXT NOT NULL DEFAULT '{}'
);
INSERT INTO webauthn_sessions_new (
id,
created_at,
challenge,
expires_at,
user_verification,
credential_params,
extensions
)
SELECT
id,
created_at,
challenge,
expires_at,
user_verification,
credential_params,
extensions
FROM webauthn_sessions;
DROP TABLE webauthn_sessions;
ALTER TABLE webauthn_sessions_new RENAME TO webauthn_sessions;
CREATE INDEX idx_webauthn_sessions_expires_at ON webauthn_sessions (expires_at);
CREATE TABLE oauth2_sessions_new (
id TEXT NOT NULL PRIMARY KEY,
created_at DATETIME NOT NULL,
kind TEXT NOT NULL,
key TEXT NOT NULL,
request_id TEXT NOT NULL,
client_id TEXT NOT NULL REFERENCES oidc_clients(id) ON DELETE CASCADE,
access_token_signature TEXT NOT NULL DEFAULT '',
active BOOLEAN NOT NULL DEFAULT TRUE,
request_data BLOB NOT NULL,
expires_at DATETIME,
rotated_at INTEGER,
CONSTRAINT chk_oauth2_sessions_client_id
CHECK (client_id = json_extract(CAST(request_data AS TEXT), '$.client_id'))
);
INSERT INTO oauth2_sessions_new (
id,
created_at,
kind,
key,
request_id,
client_id,
access_token_signature,
active,
request_data,
expires_at,
rotated_at
)
SELECT
id,
created_at,
kind,
key,
request_id,
client_id,
access_token_signature,
active,
request_data,
expires_at,
rotated_at
FROM oauth2_sessions;
DROP TABLE oauth2_sessions;
ALTER TABLE oauth2_sessions_new RENAME TO oauth2_sessions;
CREATE UNIQUE INDEX idx_oauth2_sessions_kind_key ON oauth2_sessions (kind, key);
CREATE INDEX idx_oauth2_sessions_kind_request ON oauth2_sessions (kind, request_id);
CREATE INDEX idx_oauth2_sessions_expires_at ON oauth2_sessions (expires_at);
CREATE INDEX idx_oauth2_sessions_client_subject
ON oauth2_sessions (client_id, json_extract(CAST(request_data AS TEXT), '$.session.subject'), kind, active);
-- Restore the TEXT primary keys without the NOT NULL constraint
CREATE TABLE api_keys_new
(
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
key TEXT NOT NULL UNIQUE,
description TEXT,
expires_at DATETIME NOT NULL,
last_used_at DATETIME,
created_at DATETIME,
user_id TEXT REFERENCES users(id) ON DELETE CASCADE,
expiration_email_sent BOOLEAN NOT NULL DEFAULT 0
);
INSERT INTO api_keys_new (
id,
name,
key,
description,
expires_at,
last_used_at,
created_at,
user_id,
expiration_email_sent
)
SELECT
id,
name,
key,
description,
expires_at,
last_used_at,
created_at,
user_id,
expiration_email_sent
FROM api_keys;
DROP TABLE api_keys;
ALTER TABLE api_keys_new RENAME TO api_keys;
CREATE INDEX idx_api_keys_key ON api_keys(key);
CREATE INDEX idx_api_keys_expires_at ON api_keys(expires_at);
CREATE TABLE audit_logs_new
(
id TEXT PRIMARY KEY,
created_at DATETIME NOT NULL,
event TEXT NOT NULL,
ip_address TEXT,
user_agent TEXT NOT NULL,
data BLOB NOT NULL,
user_id TEXT REFERENCES users ON DELETE CASCADE,
country TEXT,
city TEXT
);
INSERT INTO audit_logs_new (
id,
created_at,
event,
ip_address,
user_agent,
data,
user_id,
country,
city
)
SELECT
id,
created_at,
event,
ip_address,
user_agent,
data,
user_id,
country,
city
FROM audit_logs;
DROP TABLE audit_logs;
ALTER TABLE audit_logs_new RENAME TO audit_logs;
CREATE INDEX idx_audit_logs_client_name ON audit_logs((json_extract(data, '$.clientName')));
CREATE INDEX idx_audit_logs_country ON audit_logs (country);
CREATE INDEX idx_audit_logs_created_at ON audit_logs (created_at);
CREATE INDEX idx_audit_logs_event ON audit_logs (event);
CREATE INDEX idx_audit_logs_user_agent ON audit_logs (user_agent);
CREATE INDEX idx_audit_logs_user_id ON audit_logs (user_id);
CREATE TABLE oidc_clients_new
(
id TEXT PRIMARY KEY,
created_at DATETIME NOT NULL,
name TEXT,
callback_urls BLOB,
image_type TEXT,
created_by_id TEXT REFERENCES users ON DELETE SET NULL,
is_public BOOLEAN DEFAULT FALSE,
pkce_enabled BOOLEAN DEFAULT FALSE,
logout_callback_urls BLOB,
credentials BLOB,
launch_url TEXT,
requires_reauthentication BOOLEAN NOT NULL DEFAULT FALSE,
dark_image_type TEXT,
is_group_restricted BOOLEAN NOT NULL DEFAULT 0,
requires_pushed_authorization_requests BOOLEAN NOT NULL DEFAULT FALSE,
skip_consent BOOLEAN NOT NULL DEFAULT FALSE,
pkce_supported BOOLEAN NOT NULL DEFAULT 0,
description TEXT NOT NULL DEFAULT '',
client_type TEXT NOT NULL DEFAULT 'standard',
metadata_expires_at DATETIME,
metadata_grant_types BLOB,
access_token_duration_minutes INTEGER NOT NULL DEFAULT 60,
refresh_token_duration_minutes INTEGER NOT NULL DEFAULT 43200,
backchannel_logout_url TEXT NOT NULL DEFAULT ''
);
INSERT INTO oidc_clients_new (
id,
created_at,
name,
callback_urls,
image_type,
created_by_id,
is_public,
pkce_enabled,
logout_callback_urls,
credentials,
launch_url,
requires_reauthentication,
dark_image_type,
is_group_restricted,
requires_pushed_authorization_requests,
skip_consent,
pkce_supported,
description,
client_type,
metadata_expires_at,
metadata_grant_types,
access_token_duration_minutes,
refresh_token_duration_minutes,
backchannel_logout_url
)
SELECT
id,
created_at,
name,
callback_urls,
image_type,
created_by_id,
is_public,
pkce_enabled,
logout_callback_urls,
credentials,
launch_url,
requires_reauthentication,
dark_image_type,
is_group_restricted,
requires_pushed_authorization_requests,
skip_consent,
pkce_supported,
description,
client_type,
metadata_expires_at,
metadata_grant_types,
access_token_duration_minutes,
refresh_token_duration_minutes,
backchannel_logout_url
FROM oidc_clients;
DROP TABLE oidc_clients;
ALTER TABLE oidc_clients_new RENAME TO oidc_clients;
CREATE TABLE reauthentication_tokens_new
(
id TEXT PRIMARY KEY,
created_at DATETIME NOT NULL,
token TEXT NOT NULL UNIQUE,
expires_at DATETIME NOT NULL,
user_id TEXT NOT NULL REFERENCES users ON DELETE CASCADE
);
INSERT INTO reauthentication_tokens_new (
id,
created_at,
token,
expires_at,
user_id
)
SELECT
id,
created_at,
token,
expires_at,
user_id
FROM reauthentication_tokens;
DROP TABLE reauthentication_tokens;
ALTER TABLE reauthentication_tokens_new RENAME TO reauthentication_tokens;
CREATE INDEX idx_reauthentication_tokens_token ON reauthentication_tokens (token);
CREATE INDEX idx_reauthentication_tokens_expires_at ON reauthentication_tokens (expires_at);
CREATE TABLE scim_service_providers_new
(
id TEXT PRIMARY KEY,
created_at DATETIME NOT NULL,
endpoint TEXT NOT NULL,
token TEXT NOT NULL,
last_synced_at DATETIME,
oidc_client_id TEXT NOT NULL,
FOREIGN KEY (oidc_client_id) REFERENCES oidc_clients (id) ON DELETE CASCADE
);
INSERT INTO scim_service_providers_new (
id,
created_at,
endpoint,
token,
last_synced_at,
oidc_client_id
)
SELECT
id,
created_at,
endpoint,
token,
last_synced_at,
oidc_client_id
FROM scim_service_providers;
DROP TABLE scim_service_providers;
ALTER TABLE scim_service_providers_new RENAME TO scim_service_providers;
CREATE TABLE webauthn_credentials_new
(
id TEXT PRIMARY KEY,
created_at DATETIME NOT NULL,
name TEXT NOT NULL,
credential_id BLOB NOT NULL UNIQUE,
public_key BLOB NOT NULL,
attestation_type TEXT NOT NULL,
transport BLOB NOT NULL,
user_id TEXT REFERENCES users ON DELETE CASCADE,
backup_eligible BOOLEAN DEFAULT FALSE NOT NULL,
backup_state BOOLEAN DEFAULT FALSE NOT NULL,
aaguid TEXT NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000'
);
INSERT INTO webauthn_credentials_new (
id,
created_at,
name,
credential_id,
public_key,
attestation_type,
transport,
user_id,
backup_eligible,
backup_state,
aaguid
)
SELECT
id,
created_at,
name,
credential_id,
public_key,
attestation_type,
transport,
user_id,
backup_eligible,
backup_state,
aaguid
FROM webauthn_credentials;
DROP TABLE webauthn_credentials;
ALTER TABLE webauthn_credentials_new RENAME TO webauthn_credentials;
COMMIT;
PRAGMA foreign_keys = ON;
backend/resources/migrations/sqlite/20261006120000_sqlite_postgres_schema_alignment.up.sql
+380
View File
@@ -0,0 +1,380 @@
PRAGMA foreign_keys = OFF;
BEGIN;
-- Align the webauthn_sessions.extensions (JSONB) and oauth2_sessions.rotated_at (TIMESTAMPTZ) column types with the export format used for PostgreSQL
CREATE TABLE webauthn_sessions_new
(
id TEXT NOT NULL PRIMARY KEY,
created_at DATETIME,
challenge TEXT NOT NULL UNIQUE,
expires_at DATETIME NOT NULL,
user_verification TEXT NOT NULL,
credential_params BLOB DEFAULT '[]' NOT NULL,
extensions BLOB NOT NULL DEFAULT '{}'
);
INSERT INTO webauthn_sessions_new (
id,
created_at,
challenge,
expires_at,
user_verification,
credential_params,
extensions
)
SELECT
id,
created_at,
challenge,
expires_at,
user_verification,
credential_params,
extensions
FROM webauthn_sessions;
DROP TABLE webauthn_sessions;
ALTER TABLE webauthn_sessions_new RENAME TO webauthn_sessions;
CREATE INDEX idx_webauthn_sessions_expires_at ON webauthn_sessions (expires_at);
CREATE TABLE oauth2_sessions_new (
id TEXT NOT NULL PRIMARY KEY,
created_at DATETIME NOT NULL,
kind TEXT NOT NULL,
key TEXT NOT NULL,
request_id TEXT NOT NULL,
client_id TEXT NOT NULL REFERENCES oidc_clients(id) ON DELETE CASCADE,
access_token_signature TEXT NOT NULL DEFAULT '',
active BOOLEAN NOT NULL DEFAULT TRUE,
request_data BLOB NOT NULL,
expires_at DATETIME,
rotated_at DATETIME,
CONSTRAINT chk_oauth2_sessions_client_id
CHECK (client_id = json_extract(CAST(request_data AS TEXT), '$.client_id'))
);
INSERT INTO oauth2_sessions_new (
id,
created_at,
kind,
key,
request_id,
client_id,
access_token_signature,
active,
request_data,
expires_at,
rotated_at
)
SELECT
id,
created_at,
kind,
key,
request_id,
client_id,
access_token_signature,
active,
request_data,
expires_at,
rotated_at
FROM oauth2_sessions;
DROP TABLE oauth2_sessions;
ALTER TABLE oauth2_sessions_new RENAME TO oauth2_sessions;
CREATE UNIQUE INDEX idx_oauth2_sessions_kind_key ON oauth2_sessions (kind, key);
CREATE INDEX idx_oauth2_sessions_kind_request ON oauth2_sessions (kind, request_id);
CREATE INDEX idx_oauth2_sessions_expires_at ON oauth2_sessions (expires_at);
CREATE INDEX idx_oauth2_sessions_client_subject
ON oauth2_sessions (client_id, json_extract(CAST(request_data AS TEXT), '$.session.subject'), kind, active);
-- Declare the TEXT primary keys as NOT NULL, because SQLite (unlike PostgreSQL) otherwise accepts NULL in them
-- Rows with a NULL id cannot be referenced or addressed by the application, so they are dropped
CREATE TABLE api_keys_new
(
id TEXT NOT NULL PRIMARY KEY,
name TEXT NOT NULL,
key TEXT NOT NULL UNIQUE,
description TEXT,
expires_at DATETIME NOT NULL,
last_used_at DATETIME,
created_at DATETIME,
user_id TEXT REFERENCES users(id) ON DELETE CASCADE,
expiration_email_sent BOOLEAN NOT NULL DEFAULT 0
);
INSERT INTO api_keys_new (
id,
name,
key,
description,
expires_at,
last_used_at,
created_at,
user_id,
expiration_email_sent
)
SELECT
id,
name,
key,
description,
expires_at,
last_used_at,
created_at,
user_id,
expiration_email_sent
FROM api_keys
WHERE id IS NOT NULL;
DROP TABLE api_keys;
ALTER TABLE api_keys_new RENAME TO api_keys;
CREATE INDEX idx_api_keys_key ON api_keys(key);
CREATE INDEX idx_api_keys_expires_at ON api_keys(expires_at);
CREATE TABLE audit_logs_new
(
id TEXT NOT NULL PRIMARY KEY,
created_at DATETIME NOT NULL,
event TEXT NOT NULL,
ip_address TEXT,
user_agent TEXT NOT NULL,
data BLOB NOT NULL,
user_id TEXT REFERENCES users ON DELETE CASCADE,
country TEXT,
city TEXT
);
INSERT INTO audit_logs_new (
id,
created_at,
event,
ip_address,
user_agent,
data,
user_id,
country,
city
)
SELECT
id,
created_at,
event,
ip_address,
user_agent,
data,
user_id,
country,
city
FROM audit_logs
WHERE id IS NOT NULL;
DROP TABLE audit_logs;
ALTER TABLE audit_logs_new RENAME TO audit_logs;
CREATE INDEX idx_audit_logs_client_name ON audit_logs((json_extract(data, '$.clientName')));
CREATE INDEX idx_audit_logs_country ON audit_logs (country);
CREATE INDEX idx_audit_logs_created_at ON audit_logs (created_at);
CREATE INDEX idx_audit_logs_event ON audit_logs (event);
CREATE INDEX idx_audit_logs_user_agent ON audit_logs (user_agent);
CREATE INDEX idx_audit_logs_user_id ON audit_logs (user_id);
CREATE TABLE oidc_clients_new
(
id TEXT NOT NULL PRIMARY KEY,
created_at DATETIME NOT NULL,
name TEXT,
callback_urls BLOB,
image_type TEXT,
created_by_id TEXT REFERENCES users ON DELETE SET NULL,
is_public BOOLEAN DEFAULT FALSE,
pkce_enabled BOOLEAN DEFAULT FALSE,
logout_callback_urls BLOB,
credentials BLOB,
launch_url TEXT,
requires_reauthentication BOOLEAN NOT NULL DEFAULT FALSE,
dark_image_type TEXT,
is_group_restricted BOOLEAN NOT NULL DEFAULT 0,
requires_pushed_authorization_requests BOOLEAN NOT NULL DEFAULT FALSE,
skip_consent BOOLEAN NOT NULL DEFAULT FALSE,
pkce_supported BOOLEAN NOT NULL DEFAULT 0,
description TEXT NOT NULL DEFAULT '',
client_type TEXT NOT NULL DEFAULT 'standard',
metadata_expires_at DATETIME,
metadata_grant_types BLOB,
access_token_duration_minutes INTEGER NOT NULL DEFAULT 60,
refresh_token_duration_minutes INTEGER NOT NULL DEFAULT 43200,
backchannel_logout_url TEXT NOT NULL DEFAULT ''
);
INSERT INTO oidc_clients_new (
id,
created_at,
name,
callback_urls,
image_type,
created_by_id,
is_public,
pkce_enabled,
logout_callback_urls,
credentials,
launch_url,
requires_reauthentication,
dark_image_type,
is_group_restricted,
requires_pushed_authorization_requests,
skip_consent,
pkce_supported,
description,
client_type,
metadata_expires_at,
metadata_grant_types,
access_token_duration_minutes,
refresh_token_duration_minutes,
backchannel_logout_url
)
SELECT
id,
created_at,
name,
callback_urls,
image_type,
created_by_id,
is_public,
pkce_enabled,
logout_callback_urls,
credentials,
launch_url,
requires_reauthentication,
dark_image_type,
is_group_restricted,
requires_pushed_authorization_requests,
skip_consent,
pkce_supported,
description,
client_type,
metadata_expires_at,
metadata_grant_types,
access_token_duration_minutes,
refresh_token_duration_minutes,
backchannel_logout_url
FROM oidc_clients
WHERE id IS NOT NULL;
DROP TABLE oidc_clients;
ALTER TABLE oidc_clients_new RENAME TO oidc_clients;
CREATE TABLE reauthentication_tokens_new
(
id TEXT NOT NULL PRIMARY KEY,
created_at DATETIME NOT NULL,
token TEXT NOT NULL UNIQUE,
expires_at DATETIME NOT NULL,
user_id TEXT NOT NULL REFERENCES users ON DELETE CASCADE
);
INSERT INTO reauthentication_tokens_new (
id,
created_at,
token,
expires_at,
user_id
)
SELECT
id,
created_at,
token,
expires_at,
user_id
FROM reauthentication_tokens
WHERE id IS NOT NULL;
DROP TABLE reauthentication_tokens;
ALTER TABLE reauthentication_tokens_new RENAME TO reauthentication_tokens;
CREATE INDEX idx_reauthentication_tokens_token ON reauthentication_tokens (token);
CREATE INDEX idx_reauthentication_tokens_expires_at ON reauthentication_tokens (expires_at);
CREATE TABLE scim_service_providers_new
(
id TEXT NOT NULL PRIMARY KEY,
created_at DATETIME NOT NULL,
endpoint TEXT NOT NULL,
token TEXT NOT NULL,
last_synced_at DATETIME,
oidc_client_id TEXT NOT NULL,
FOREIGN KEY (oidc_client_id) REFERENCES oidc_clients (id) ON DELETE CASCADE
);
INSERT INTO scim_service_providers_new (
id,
created_at,
endpoint,
token,
last_synced_at,
oidc_client_id
)
SELECT
id,
created_at,
endpoint,
token,
last_synced_at,
oidc_client_id
FROM scim_service_providers
WHERE id IS NOT NULL;
DROP TABLE scim_service_providers;
ALTER TABLE scim_service_providers_new RENAME TO scim_service_providers;
CREATE TABLE webauthn_credentials_new
(
id TEXT NOT NULL PRIMARY KEY,
created_at DATETIME NOT NULL,
name TEXT NOT NULL,
credential_id BLOB NOT NULL UNIQUE,
public_key BLOB NOT NULL,
attestation_type TEXT NOT NULL,
transport BLOB NOT NULL,
user_id TEXT REFERENCES users ON DELETE CASCADE,
backup_eligible BOOLEAN DEFAULT FALSE NOT NULL,
backup_state BOOLEAN DEFAULT FALSE NOT NULL,
aaguid TEXT NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000'
);
INSERT INTO webauthn_credentials_new (
id,
created_at,
name,
credential_id,
public_key,
attestation_type,
transport,
user_id,
backup_eligible,
backup_state,
aaguid
)
SELECT
id,
created_at,
name,
credential_id,
public_key,
attestation_type,
transport,
user_id,
backup_eligible,
backup_state,
aaguid
FROM webauthn_credentials
WHERE id IS NOT NULL;
DROP TABLE webauthn_credentials;
ALTER TABLE webauthn_credentials_new RENAME TO webauthn_credentials;
COMMIT;
PRAGMA foreign_keys = ON;