mirror of
https://github.com/netbirdio/netbird.git
synced 2026-10-21 04:49:10 +02:00
[management] Remove the MySQL store engine (#7914)
This commit is contained in:
@@ -34,7 +34,7 @@ jobs:
|
||||
while IFS= read -r dir; do
|
||||
echo "=== Checking $dir ==="
|
||||
# Search for problematic imports, excluding test files
|
||||
RESULTS=$(grep -r "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\)" "$dir" --include="*.go" 2>/dev/null | grep -v "_test.go" | grep -v "test_" | grep -v "/test/" | grep -v "tools/idp-migrate/" | grep -v "tools/mysql-migrate/" || true)
|
||||
RESULTS=$(grep -r "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\)" "$dir" --include="*.go" 2>/dev/null | grep -v "_test.go" | grep -v "test_" | grep -v "/test/" | grep -v "tools/idp-migrate/" || true)
|
||||
if [ -n "$RESULTS" ]; then
|
||||
echo "❌ Found problematic dependencies:"
|
||||
echo "$RESULTS"
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
IMPORTERS=$(go list -json -deps ./... 2>/dev/null | jq -r "select(.Imports[]? == \"$package\") | .ImportPath")
|
||||
|
||||
# Check if any importer is NOT in management/signal/relay
|
||||
BSD_IMPORTER=$(echo "$IMPORTERS" | grep -v "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\|combined\|tools/idp-migrate\|tools/mysql-migrate\)" | head -1)
|
||||
BSD_IMPORTER=$(echo "$IMPORTERS" | grep -v "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\|combined\|tools/idp-migrate\)" | head -1)
|
||||
|
||||
if [ -n "$BSD_IMPORTER" ]; then
|
||||
echo "❌ $package ($license) is imported by BSD-licensed code: $BSD_IMPORTER"
|
||||
|
||||
@@ -307,7 +307,7 @@ jobs:
|
||||
sh -c ' \
|
||||
apk update; apk add --no-cache \
|
||||
ca-certificates iptables ip6tables dbus dbus-dev libpcap-dev build-base; \
|
||||
go test -buildvcs=false -tags "devcert privileged" -v -timeout 10m -p 1 $(go list -e -buildvcs=false ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/ui -e /upload-server -e /client/testutil/privileged -e tools/mysql-migrate)
|
||||
go test -buildvcs=false -tags "devcert privileged" -v -timeout 10m -p 1 $(go list -e -buildvcs=false ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/ui -e /upload-server -e /client/testutil/privileged)
|
||||
'
|
||||
|
||||
test_relay:
|
||||
@@ -498,7 +498,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
arch: ["amd64"]
|
||||
store: ["sqlite", "postgres", "mysql"]
|
||||
store: ["sqlite", "postgres"]
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -547,10 +547,6 @@ jobs:
|
||||
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
|
||||
run: echo "$DOCKER_TOKEN" | sudo docker login --username "$DOCKER_USER" --password-stdin
|
||||
|
||||
- name: download mysql image
|
||||
if: matrix.store == 'mysql'
|
||||
run: docker pull mlsmaycon/warmed-mysql:8
|
||||
|
||||
# The -json stream goes through tools/gotestsummary so the log shows one
|
||||
# line per test, the output of failed tests, the head of a timeout panic
|
||||
# with the still-running tests, and the slowest tests per package.
|
||||
|
||||
@@ -76,7 +76,7 @@ jobs:
|
||||
# resolve; the Where-Object pipeline then drops the broken package by
|
||||
# path. Without -e, go list aborts with empty stdout.
|
||||
run: |
|
||||
$packages = go list -e ./... | Where-Object { $_ -notmatch '/management' } | Where-Object { $_ -notmatch '/relay' } | Where-Object { $_ -notmatch '/signal' } | Where-Object { $_ -notmatch '/proxy' } | Where-Object { $_ -notmatch '/combined' } | Where-Object { $_ -notmatch '/client/ui$' } | Where-Object { $_ -notmatch '/tools/mysql-migrate' }
|
||||
$packages = go list -e ./... | Where-Object { $_ -notmatch '/management' } | Where-Object { $_ -notmatch '/relay' } | Where-Object { $_ -notmatch '/signal' } | Where-Object { $_ -notmatch '/proxy' } | Where-Object { $_ -notmatch '/combined' } | Where-Object { $_ -notmatch '/client/ui$' }
|
||||
$goExe = "C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe"
|
||||
$cmd = "$goExe test -tags `"devcert privileged`" -timeout 10m -p 1 $($packages -join ' ') > test-out.txt 2>&1"
|
||||
Set-Content -Path "${{ github.workspace }}\run-tests.cmd" -Value $cmd
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
store: ["sqlite", "postgres", "mysql"]
|
||||
store: ["sqlite", "postgres"]
|
||||
services:
|
||||
postgres:
|
||||
image: ${{ (matrix.store == 'postgres') && 'postgres' || '' }}
|
||||
@@ -35,19 +35,6 @@ jobs:
|
||||
--health-timeout 5s
|
||||
ports:
|
||||
- 5432:5432
|
||||
mysql:
|
||||
image: ${{ (matrix.store == 'mysql') && 'mysql' || '' }}
|
||||
env:
|
||||
MYSQL_USER: netbird
|
||||
MYSQL_PASSWORD: mysql
|
||||
MYSQL_ROOT_PASSWORD: mysqlroot
|
||||
MYSQL_DATABASE: netbird
|
||||
options: >-
|
||||
--health-cmd "mysqladmin ping --silent"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
ports:
|
||||
- 3306:3306
|
||||
steps:
|
||||
- name: Set Database Connection String
|
||||
run: |
|
||||
@@ -56,11 +43,6 @@ jobs:
|
||||
else
|
||||
echo "NETBIRD_STORE_ENGINE_POSTGRES_DSN==" >> $GITHUB_ENV
|
||||
fi
|
||||
if [ "${{ matrix.store }}" == "mysql" ]; then
|
||||
echo "NETBIRD_STORE_ENGINE_MYSQL_DSN=netbird:mysql@tcp($(hostname -I | awk '{print $1}'):3306)/netbird" >> $GITHUB_ENV
|
||||
else
|
||||
echo "NETBIRD_STORE_ENGINE_MYSQL_DSN==" >> $GITHUB_ENV
|
||||
fi
|
||||
|
||||
- name: Install jq
|
||||
run: sudo apt-get install -y jq
|
||||
@@ -86,15 +68,6 @@ jobs:
|
||||
restore-keys: |
|
||||
${{ runner.os }}-go-
|
||||
|
||||
- name: Setup MySQL privileges
|
||||
if: matrix.store == 'mysql'
|
||||
run: |
|
||||
sleep 10
|
||||
mysql -h 127.0.0.1 -u root -pmysqlroot -e "
|
||||
GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO 'netbird'@'%';
|
||||
FLUSH PRIVILEGES;
|
||||
"
|
||||
|
||||
- name: cp setup.env
|
||||
run: cp infrastructure_files/tests/setup.env infrastructure_files/
|
||||
|
||||
@@ -114,7 +87,6 @@ jobs:
|
||||
CI_NETBIRD_AUTH_SUPPORTED_SCOPES: "openid profile email offline_access api email_verified"
|
||||
CI_NETBIRD_STORE_CONFIG_ENGINE: ${{ matrix.store }}
|
||||
NETBIRD_STORE_ENGINE_POSTGRES_DSN: ${{ env.NETBIRD_STORE_ENGINE_POSTGRES_DSN }}
|
||||
NETBIRD_STORE_ENGINE_MYSQL_DSN: ${{ env.NETBIRD_STORE_ENGINE_MYSQL_DSN }}
|
||||
CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false
|
||||
|
||||
- name: check values
|
||||
@@ -143,7 +115,6 @@ jobs:
|
||||
CI_NETBIRD_SIGNAL_PORT: 12345
|
||||
CI_NETBIRD_STORE_CONFIG_ENGINE: ${{ matrix.store }}
|
||||
NETBIRD_STORE_ENGINE_POSTGRES_DSN: "${{ env.NETBIRD_STORE_ENGINE_POSTGRES_DSN }}$"
|
||||
NETBIRD_STORE_ENGINE_MYSQL_DSN: "${{ env.NETBIRD_STORE_ENGINE_MYSQL_DSN }}$"
|
||||
CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false
|
||||
CI_NETBIRD_TURN_EXTERNAL_IP: "1.2.3.4"
|
||||
CI_NETBIRD_MGMT_DISABLE_DEFAULT_POLICY: false
|
||||
@@ -182,7 +153,6 @@ jobs:
|
||||
grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep Scope | grep "$CI_NETBIRD_AUTH_SUPPORTED_SCOPES"
|
||||
grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep -A 3 RedirectURLs | grep "http://localhost:53000"
|
||||
grep "external-ip" turnserver.conf | grep $CI_NETBIRD_TURN_EXTERNAL_IP
|
||||
grep "NETBIRD_STORE_ENGINE_MYSQL_DSN=$NETBIRD_STORE_ENGINE_MYSQL_DSN" docker-compose.yml
|
||||
grep NETBIRD_STORE_ENGINE_POSTGRES_DSN docker-compose.yml | egrep "$NETBIRD_STORE_ENGINE_POSTGRES_DSN"
|
||||
# check relay values
|
||||
grep "NB_EXPOSED_ADDRESS=rels://$CI_NETBIRD_DOMAIN:33445" docker-compose.yml
|
||||
|
||||
@@ -280,28 +280,6 @@ builds:
|
||||
- -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
|
||||
mod_timestamp: "{{ .CommitTimestamp }}"
|
||||
|
||||
- id: netbird-mysql-migrate
|
||||
dir: tools/mysql-migrate
|
||||
env:
|
||||
- CGO_ENABLED=1
|
||||
- >-
|
||||
{{- if eq .Runtime.Goos "linux" }}
|
||||
{{- if eq .Arch "arm64"}}CC=aarch64-linux-gnu-gcc{{- end }}
|
||||
{{- if eq .Arch "arm"}}CC=arm-linux-gnueabihf-gcc{{- end }}
|
||||
{{- end }}
|
||||
binary: netbird-mysql-migrate
|
||||
goos:
|
||||
- linux
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
- arm
|
||||
goarm:
|
||||
- 7
|
||||
ldflags:
|
||||
- -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser
|
||||
mod_timestamp: "{{ .CommitTimestamp }}"
|
||||
|
||||
universal_binaries:
|
||||
- id: netbird
|
||||
|
||||
@@ -318,10 +296,6 @@ archives:
|
||||
builds:
|
||||
- netbird-idp-migrate
|
||||
name_template: "netbird-idp-migrate_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
|
||||
- id: netbird-mysql-migrate
|
||||
builds:
|
||||
- netbird-mysql-migrate
|
||||
name_template: "netbird-mysql-migrate_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
|
||||
- id: nblink
|
||||
builds:
|
||||
- nblink
|
||||
|
||||
@@ -35,13 +35,9 @@ func TestOpenAdminEventStoreMissingEncryptionKeyReturnsNilInterface(t *testing.T
|
||||
|
||||
func TestApplyServerStoreEnv(t *testing.T) {
|
||||
t.Setenv("NB_STORE_ENGINE_POSTGRES_DSN", "")
|
||||
t.Setenv("NB_STORE_ENGINE_MYSQL_DSN", "")
|
||||
t.Setenv("NB_STORE_ENGINE_SQLITE_FILE", "")
|
||||
|
||||
applyServerStoreEnv(StoreConfig{Engine: "postgres", DSN: "postgres-dsn", File: "store.db"})
|
||||
require.Equal(t, "postgres-dsn", os.Getenv("NB_STORE_ENGINE_POSTGRES_DSN"))
|
||||
require.Equal(t, "store.db", os.Getenv("NB_STORE_ENGINE_SQLITE_FILE"))
|
||||
|
||||
applyServerStoreEnv(StoreConfig{Engine: "mysql", DSN: "mysql-dsn"})
|
||||
require.Equal(t, "mysql-dsn", os.Getenv("NB_STORE_ENGINE_MYSQL_DSN"))
|
||||
}
|
||||
|
||||
@@ -188,7 +188,7 @@ type RelaysConfig struct {
|
||||
type StoreConfig struct {
|
||||
Engine string `yaml:"engine"`
|
||||
EncryptionKey string `yaml:"encryptionKey"`
|
||||
DSN string `yaml:"dsn"` // Connection string for postgres or mysql engines
|
||||
DSN string `yaml:"dsn"` // Connection string for the postgres engine
|
||||
File string `yaml:"file"` // SQLite database file path (optional, defaults to dataDir)
|
||||
}
|
||||
|
||||
|
||||
@@ -127,13 +127,8 @@ func execute(cmd *cobra.Command, _ []string) error {
|
||||
|
||||
// initializeConfig loads and validates the configuration, then initializes logging.
|
||||
func applyServerStoreEnv(storeConfig StoreConfig) {
|
||||
if dsn := storeConfig.DSN; dsn != "" {
|
||||
switch strings.ToLower(storeConfig.Engine) {
|
||||
case "postgres":
|
||||
os.Setenv("NB_STORE_ENGINE_POSTGRES_DSN", dsn)
|
||||
case "mysql":
|
||||
os.Setenv("NB_STORE_ENGINE_MYSQL_DSN", dsn)
|
||||
}
|
||||
if dsn := storeConfig.DSN; dsn != "" && strings.EqualFold(storeConfig.Engine, "postgres") {
|
||||
os.Setenv("NB_STORE_ENGINE_POSTGRES_DSN", dsn)
|
||||
}
|
||||
if file := storeConfig.File; file != "" {
|
||||
os.Setenv("NB_STORE_ENGINE_SQLITE_FILE", file)
|
||||
|
||||
@@ -110,8 +110,8 @@ server:
|
||||
|
||||
# Store configuration
|
||||
store:
|
||||
engine: "sqlite" # sqlite, postgres, or mysql
|
||||
dsn: "" # Connection string for postgres or mysql
|
||||
engine: "sqlite" # sqlite or postgres
|
||||
dsn: "" # Connection string for postgres
|
||||
encryptionKey: ""
|
||||
# file: "" # Custom SQLite file path (optional, defaults to {dataDir}/store.db)
|
||||
|
||||
|
||||
@@ -64,17 +64,17 @@ Reads decrypt provider secrets in-place; writes do `provider.Copy().EncryptSensi
|
||||
|
||||
### Correctness
|
||||
- `SaveAgentNetworkProvider` saves the copy (sql_store_agentnetwork.go:95). The caller's in-memory pointer therefore keeps plaintext `api_key` and any `CreatedAt`/`UpdatedAt` gorm autofills land on the copy, not the original. Callers that need synced timestamps must re-fetch.
|
||||
- `IncrementAgentNetworkConsumption`'s `Create` provides initial counter values (`TokensInput: tokensIn`, etc.) in the row, and on conflict the assignments add the same deltas to the existing values. The insert-vs-update arithmetic is consistent. Cross-check that no engine in use (sqlite, postgres, mysql) silently rejects the `OnConflict` clause — GORM emits engine-specific SQL but `ON DUPLICATE KEY UPDATE` (mysql) vs `ON CONFLICT (...)` (sqlite/postgres) need their unique constraint to match the composite PK on `agent_network_consumption`; it does, by construction.
|
||||
- `IncrementAgentNetworkConsumption`'s `Create` provides initial counter values (`TokensInput: tokensIn`, etc.) in the row, and on conflict the assignments add the same deltas to the existing values. The insert-vs-update arithmetic is consistent. Cross-check that no engine in use (sqlite, postgres) silently rejects the `OnConflict` clause — GORM emits `ON CONFLICT (...)`, which needs its unique constraint to match the composite PK on `agent_network_consumption`; it does, by construction.
|
||||
- `IncrementAgentNetworkConsumption` writes `updated_at: time.Now().UTC()` literally inside the assignments map (sql_store_agentnetwork.go:333) — fine, but it's a Go-side timestamp captured at call time, not a DB-side `now()`. Acceptable for an audit field.
|
||||
- `GetAgentNetworkConsumption` returns a zero-valued non-nil row on `ErrRecordNotFound` (sql_store_agentnetwork.go:364-371). Document or rename — a typed sentinel error would be more orthodox; callers must know not to error-check.
|
||||
|
||||
### Concurrency / transactions
|
||||
- Hot-path `IncrementAgentNetworkConsumption` runs outside any explicit transaction; concurrency safety relies entirely on the DB serialising the `ON CONFLICT` upsert against the composite PK. This is correct for postgres and mysql; for sqlite it serialises behind the single writer.
|
||||
- Hot-path `IncrementAgentNetworkConsumption` runs outside any explicit transaction; concurrency safety relies entirely on the DB serialising the `ON CONFLICT` upsert against the composite PK. This is correct for postgres; for sqlite it serialises behind the single writer.
|
||||
- `SaveAgentNetworkSettings` is a blind upsert with no version/etag — concurrent writes from two operators last-write-wins on the collection-toggle flags (settings.go:23-25). Acceptable for admin-curated state but worth flagging.
|
||||
- `Save*Provider` uses `db.Save` on a struct with a PK already set — GORM emits UPDATE or INSERT based on row existence. No upsert clause is attached, so a race between two creates with the same generated `xid` (vanishingly unlikely) would surface as a PK violation.
|
||||
|
||||
### Migration safety
|
||||
- All six tables ride `AutoMigrate` (sql_store.go:141-142). AutoMigrate is additive: new columns get added, but it never drops columns nor narrows types. Three `bool` columns on `agent_network_settings` (`EnableLogCollection`, `EnablePromptCollection`, `RedactPii`) default to false at the GORM/DDL layer for existing rows; the test at sql_store_agentnetwork_budgetrule_test.go:83-112 locks that down on a fresh sqlite. Verify postgres/mysql produce the same default.
|
||||
- All six tables ride `AutoMigrate` (sql_store.go:141-142). AutoMigrate is additive: new columns get added, but it never drops columns nor narrows types. Three `bool` columns on `agent_network_settings` (`EnableLogCollection`, `EnablePromptCollection`, `RedactPii`) default to false at the GORM/DDL layer for existing rows; the test at sql_store_agentnetwork_budgetrule_test.go:83-112 locks that down on a fresh sqlite. Verify postgres produces the same default.
|
||||
- The named index `idx_agent_network_settings_cluster_subdomain` on settings.go:15 is declared on only `subdomain`. Either the cluster column also needs `gorm:"index:idx_agent_network_settings_cluster_subdomain"` to make it composite, or the name is misleading.
|
||||
- The named index `idx_agent_network_provider` on `Provider.ProviderID` (provider.go:30) is *not* unique and not scoped to account — two providers in the same account with the same `provider_id` are permitted at the DB layer; uniqueness, if any, must live above the store.
|
||||
|
||||
@@ -101,7 +101,7 @@ Gap: there is no store-level test for providers (encryption round-trip), policie
|
||||
|
||||
- No retention / GC for `agent_network_consumption`.
|
||||
- No `Delete` for `Settings` (one row per account, cleared with the account).
|
||||
- No DB-engine-specific tuning — the same struct tags drive sqlite, mysql, postgres.
|
||||
- No DB-engine-specific tuning — the same struct tags drive sqlite and postgres.
|
||||
- Provider `extra_values` and `models` are JSON blobs; querying inside them is not supported by design.
|
||||
- `GetAgentNetworkConsumption` "not-found = zero row" contract is convenient but unconventional.
|
||||
|
||||
|
||||
@@ -113,7 +113,6 @@ require (
|
||||
github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8
|
||||
github.com/stretchr/testify v1.12.1
|
||||
github.com/testcontainers/testcontainers-go v0.37.0
|
||||
github.com/testcontainers/testcontainers-go/modules/mysql v0.37.0
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.37.0
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.37.0
|
||||
github.com/things-go/go-socks5 v0.0.4
|
||||
@@ -142,7 +141,6 @@ require (
|
||||
google.golang.org/api v0.276.0
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
gorm.io/driver/mysql v1.5.7
|
||||
gorm.io/driver/postgres v1.5.7
|
||||
gorm.io/driver/sqlite v1.5.7
|
||||
gorm.io/gorm v1.25.12
|
||||
|
||||
@@ -252,7 +252,6 @@ github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3yg
|
||||
github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
|
||||
github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6 h1:teYtXy9B7y5lHTp8V9KPxpYRAVA7dozigQcMiBust1s=
|
||||
github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI=
|
||||
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
|
||||
@@ -721,8 +720,6 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/testcontainers/testcontainers-go v0.37.0 h1:L2Qc0vkTw2EHWQ08djon0D2uw7Z/PtHS/QzZZ5Ra/hg=
|
||||
github.com/testcontainers/testcontainers-go v0.37.0/go.mod h1:QPzbxZhQ6Bclip9igjLFj6z0hs01bU8lrl2dHQmgFGM=
|
||||
github.com/testcontainers/testcontainers-go/modules/mysql v0.37.0 h1:LqUos1oR5iuuzorFnSvxsHNdYdCHB/DfI82CuT58wbI=
|
||||
github.com/testcontainers/testcontainers-go/modules/mysql v0.37.0/go.mod h1:vHEEHx5Kf+uq5hveaVAMrTzPY8eeRZcKcl23MRw5Tkc=
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.37.0 h1:hsVwFkS6s+79MbKEO+W7A1wNIw1fmkMtF4fg83m6kbc=
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.37.0/go.mod h1:Qj/eGbRbO/rEYdcRLmN+bEojzatP/+NS1y8ojl2PQsc=
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.37.0 h1:9HIY28I9ME/Zmb+zey1p/I1mto5+5ch0wLX+nJdOsQ4=
|
||||
@@ -1025,13 +1022,10 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
|
||||
gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
|
||||
gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM=
|
||||
gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA=
|
||||
gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I=
|
||||
gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4=
|
||||
gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
|
||||
gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
|
||||
gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
|
||||
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
|
||||
|
||||
@@ -53,18 +53,6 @@ if [[ "$NETBIRD_STORE_CONFIG_ENGINE" == "postgres" ]]; then
|
||||
export NETBIRD_STORE_ENGINE_POSTGRES_DSN
|
||||
fi
|
||||
|
||||
# Check if MySQL is set as the store engine
|
||||
if [[ "$NETBIRD_STORE_CONFIG_ENGINE" == "mysql" ]]; then
|
||||
# Exit if 'NETBIRD_STORE_ENGINE_MYSQL_DSN' is not set
|
||||
if [[ -z "$NETBIRD_STORE_ENGINE_MYSQL_DSN" ]]; then
|
||||
echo "Warning: NETBIRD_STORE_CONFIG_ENGINE=mysql but NETBIRD_STORE_ENGINE_MYSQL_DSN is not set."
|
||||
echo "Please add the following line to your setup.env file:"
|
||||
echo 'NETBIRD_STORE_ENGINE_MYSQL_DSN="<username>:<password>@tcp(127.0.0.1:3306)/<database>"'
|
||||
exit 1
|
||||
fi
|
||||
export NETBIRD_STORE_ENGINE_MYSQL_DSN
|
||||
fi
|
||||
|
||||
# local development or tests
|
||||
if [[ $NETBIRD_DOMAIN == "localhost" || $NETBIRD_DOMAIN == "127.0.0.1" ]]; then
|
||||
export NETBIRD_MGMT_SINGLE_ACCOUNT_MODE_DOMAIN="netbird.selfhosted"
|
||||
|
||||
@@ -94,7 +94,6 @@ services:
|
||||
]
|
||||
environment:
|
||||
- NETBIRD_STORE_ENGINE_POSTGRES_DSN=$NETBIRD_STORE_ENGINE_POSTGRES_DSN
|
||||
- NETBIRD_STORE_ENGINE_MYSQL_DSN=$NETBIRD_STORE_ENGINE_MYSQL_DSN
|
||||
|
||||
# Coturn
|
||||
coturn:
|
||||
|
||||
@@ -100,7 +100,6 @@ services:
|
||||
- traefik.http.services.netbird-management.loadbalancer.server.scheme=h2c
|
||||
environment:
|
||||
- NETBIRD_STORE_ENGINE_POSTGRES_DSN=$NETBIRD_STORE_ENGINE_POSTGRES_DSN
|
||||
- NETBIRD_STORE_ENGINE_MYSQL_DSN=$NETBIRD_STORE_ENGINE_MYSQL_DSN
|
||||
|
||||
# Coturn
|
||||
coturn:
|
||||
|
||||
@@ -855,7 +855,7 @@ configure_sqlite_store() {
|
||||
POSTGRES_DSN="host=postgres user=netbird password=${POSTGRES_PASSWORD} dbname=netbird port=5432 sslmode=disable"
|
||||
}
|
||||
|
||||
# mysql, or something this script has never seen. Swapping the images is still
|
||||
# An engine this script has never seen. Swapping the images is still
|
||||
# valid; touching the store is not.
|
||||
configure_unsupported_store() {
|
||||
MIGRATE_POSTGRES="no"
|
||||
|
||||
@@ -472,12 +472,6 @@ detect_store_config() {
|
||||
if [[ -n "$pg_dsn" ]]; then
|
||||
STORE_DSN="$pg_dsn"
|
||||
fi
|
||||
|
||||
local mysql_dsn
|
||||
mysql_dsn=$(grep '^NETBIRD_STORE_ENGINE_MYSQL_DSN=' "$INSTALL_DIR/setup.env" 2>/dev/null | sed 's/^NETBIRD_STORE_ENGINE_MYSQL_DSN=//' | tr -d '"' || echo "")
|
||||
if [[ -n "$mysql_dsn" ]]; then
|
||||
STORE_DSN="$mysql_dsn"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Also check base.setup.env
|
||||
@@ -487,12 +481,6 @@ detect_store_config() {
|
||||
if [[ -n "$pg_dsn" ]]; then
|
||||
STORE_DSN="$pg_dsn"
|
||||
fi
|
||||
|
||||
local mysql_dsn
|
||||
mysql_dsn=$(grep '^NETBIRD_STORE_ENGINE_MYSQL_DSN=' "$INSTALL_DIR/base.setup.env" 2>/dev/null | sed 's/^NETBIRD_STORE_ENGINE_MYSQL_DSN=//' | tr -d '"' || echo "")
|
||||
if [[ -n "$mysql_dsn" ]]; then
|
||||
STORE_DSN="$mysql_dsn"
|
||||
fi
|
||||
fi
|
||||
|
||||
log_success "Store engine: $STORE_ENGINE"
|
||||
|
||||
@@ -341,7 +341,7 @@ func TestGetAgentConfigForUser_RealStore(t *testing.T) {
|
||||
require.NoError(t, s.SaveAgentNetworkPolicy(ctx, newSynthTestPolicy(provider.ID, "grp-eng", "")))
|
||||
|
||||
// users.account_id is a foreign key into accounts, enforced on
|
||||
// MySQL/Postgres, so the account row must exist before its users.
|
||||
// Postgres, so the account row must exist before its users.
|
||||
require.NoError(t, s.SaveAccount(ctx, &nbtypes.Account{Id: testAccountID}))
|
||||
require.NoError(t, s.SaveUser(ctx, &nbtypes.User{
|
||||
Id: "user-in", AccountID: testAccountID, Role: nbtypes.UserRoleUser, AutoGroups: []string{"grp-eng"},
|
||||
|
||||
@@ -1277,7 +1277,6 @@ func isUniqueConstraintError(err error) bool {
|
||||
}
|
||||
msg := err.Error()
|
||||
return strings.Contains(msg, "(SQLSTATE 23505)") || // postgres
|
||||
strings.Contains(msg, "Error 1062 (23000)") || // mysql
|
||||
strings.Contains(msg, "UNIQUE constraint failed") // sqlite
|
||||
}
|
||||
|
||||
|
||||
@@ -98,9 +98,8 @@ func TestGetProvider_RedactsForReadOnlyViewer(t *testing.T) {
|
||||
// keeps the account-wide view with full config.
|
||||
|
||||
// newSelfScopeStore seeds the account and its users only, so each test
|
||||
// declares exactly the providers and policies it asserts on — the store
|
||||
// rejects re-saving a policy id on MySQL, so tests never overwrite each
|
||||
// other's rows.
|
||||
// declares exactly the providers and policies it asserts on, so tests never
|
||||
// overwrite each other's rows.
|
||||
func newSelfScopeStore(t *testing.T) (*managerImpl, store.Store) {
|
||||
t.Helper()
|
||||
mgr, s := newAgentConfigTestMgr(t)
|
||||
|
||||
@@ -21,8 +21,7 @@ type Settings struct {
|
||||
AccountID string `gorm:"primaryKey"`
|
||||
|
||||
// Domain is the gateway endpoint hostname agents call. Globally unique
|
||||
// across accounts. Sized explicitly because MySQL cannot index an
|
||||
// unbounded TEXT column; 255 covers the RFC 1035 253-octet bound.
|
||||
// across accounts. 255 covers the RFC 1035 253-octet bound.
|
||||
Domain string `gorm:"type:varchar(255);uniqueIndex:idx_agent_network_settings_domain"`
|
||||
|
||||
// ProxyAddress is the declared cluster address of the proxy serving this
|
||||
|
||||
@@ -95,7 +95,7 @@ func bucketStart(t time.Time, g UsageGranularity) time.Time {
|
||||
// AggregateUsageByGranularity buckets the usage rows by the requested
|
||||
// granularity and returns the buckets ordered oldest-first. Aggregation is done
|
||||
// in Go (rather than per-engine SQL date_trunc) so granularities stay portable
|
||||
// across SQLite/Postgres/MySQL and easy to extend.
|
||||
// across SQLite and Postgres and easy to extend.
|
||||
func AggregateUsageByGranularity(rows []*AgentNetworkUsage, g UsageGranularity) []*AgentNetworkUsageBucket {
|
||||
byPeriod := make(map[string]*AgentNetworkUsageBucket)
|
||||
for _, r := range rows {
|
||||
|
||||
@@ -6,5 +6,4 @@ type Engine string
|
||||
const (
|
||||
SqliteStoreEngine Engine = "sqlite"
|
||||
PostgresStoreEngine Engine = "postgres"
|
||||
MysqlStoreEngine Engine = "mysql"
|
||||
)
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
@@ -121,30 +120,6 @@ func OpenPostgres(ctx context.Context, dsn string, pool PoolConfig) (*Conn, erro
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
// MysqlDSN adds the connection parameters every MySQL handle needs, keeping
|
||||
// the options already present in dsn.
|
||||
func MysqlDSN(dsn string) string {
|
||||
separator := "?"
|
||||
if strings.Contains(dsn, "?") {
|
||||
separator = "&"
|
||||
}
|
||||
return dsn + separator + "charset=utf8&parseTime=True&loc=Local"
|
||||
}
|
||||
|
||||
// OpenMysql opens a MySQL database through gorm.
|
||||
func OpenMysql(ctx context.Context, dsn string) (*Conn, error) {
|
||||
gormDB, err := gorm.Open(mysql.Open(MysqlDSN(dsn)), GormConfig())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
conn, err := NewConn(ctx, gormDB, MysqlStoreEngine, nil)
|
||||
if err != nil {
|
||||
closeGorm(gormDB)
|
||||
return nil, err
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func newPgxPool(ctx context.Context, dsn string, cfg PoolConfig) (*pgxpool.Pool, error) {
|
||||
config, err := pgxpool.ParseConfig(dsn)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestMysqlDSN(t *testing.T) {
|
||||
assert.Equal(t, "user:pw@tcp(host:3306)/db?charset=utf8&parseTime=True&loc=Local", MysqlDSN("user:pw@tcp(host:3306)/db"))
|
||||
assert.Equal(t, "user:pw@tcp(host:3306)/db?tls=true&charset=utf8&parseTime=True&loc=Local", MysqlDSN("user:pw@tcp(host:3306)/db?tls=true"))
|
||||
}
|
||||
@@ -39,10 +39,7 @@ func (c *Conn) RunInTx(ctx context.Context, fn func(tx *Tx) error) error {
|
||||
return err
|
||||
}
|
||||
|
||||
err := c.withForeignKeyChecksDisabled(tx, func() error {
|
||||
return fn(&Tx{db: tx})
|
||||
})
|
||||
if err != nil {
|
||||
if err := fn(&Tx{db: tx}); err != nil {
|
||||
tx.Rollback()
|
||||
c.logIfTimedOut(ctx, timeoutCtx, err, "transaction", startTime)
|
||||
return err
|
||||
@@ -73,31 +70,6 @@ func (c *Conn) applyStatementTimeouts(tx *gorm.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// withForeignKeyChecksDisabled runs fn with MySQL's FK checks off, which avoids
|
||||
// deadlocks on MySQL and Aurora without needing SUPER privilege. The setting is
|
||||
// session-scoped and survives a rollback, so it is turned back on whenever fn
|
||||
// returns or panics; otherwise the pooled connection would keep it disabled.
|
||||
func (c *Conn) withForeignKeyChecksDisabled(tx *gorm.DB, fn func() error) (err error) {
|
||||
if c.engine != MysqlStoreEngine {
|
||||
return fn()
|
||||
}
|
||||
if err := tx.Exec("SET FOREIGN_KEY_CHECKS = 0").Error; err != nil {
|
||||
return fmt.Errorf("failed to disable FK checks: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
restoreErr := tx.Exec("SET FOREIGN_KEY_CHECKS = 1").Error
|
||||
if restoreErr == nil {
|
||||
return
|
||||
}
|
||||
if err == nil {
|
||||
err = fmt.Errorf("failed to re-enable FK checks: %w", restoreErr)
|
||||
return
|
||||
}
|
||||
log.WithContext(tx.Statement.Context).Warnf("failed to re-enable FK checks after failed transaction: %v", restoreErr)
|
||||
}()
|
||||
return fn()
|
||||
}
|
||||
|
||||
func (c *Conn) logIfTimedOut(ctx, timeoutCtx context.Context, err error, phase string, startTime time.Time) {
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(timeoutCtx.Err(), context.DeadlineExceeded) {
|
||||
log.WithContext(ctx).Warnf("%s exceeded %s timeout after %v, stack: %s", phase, c.txTimeout, time.Since(startTime), debug.Stack())
|
||||
|
||||
@@ -152,7 +152,6 @@ func (am *DefaultAccountManager) runAccountDeletionHooks(ctx context.Context, ac
|
||||
func isUniqueConstraintError(err error) bool {
|
||||
switch {
|
||||
case strings.Contains(err.Error(), "(SQLSTATE 23505)"),
|
||||
strings.Contains(err.Error(), "Error 1062 (23000)"),
|
||||
strings.Contains(err.Error(), "UNIQUE constraint failed"):
|
||||
return true
|
||||
|
||||
|
||||
@@ -3759,7 +3759,7 @@ func setupNetworkMapTest(t *testing.T) (*DefaultAccountManager, *update_channel.
|
||||
|
||||
// peerUpdateTimeout bounds how long peerShouldReceiveUpdate and its outer
|
||||
// wrappers wait for an expected update message. Sized for slow CI runners
|
||||
// (MySQL, FreeBSD, loaded sqlite) where the channel publish can take
|
||||
// (FreeBSD, loaded sqlite) where the channel publish can take
|
||||
// seconds. Only runs down on failure; passing tests return immediately
|
||||
// when the channel delivers.
|
||||
const peerUpdateTimeout = 5 * time.Second
|
||||
|
||||
@@ -429,10 +429,6 @@ func Test_SyncStatusRace(t *testing.T) {
|
||||
if os.Getenv("NETBIRD_STORE_ENGINE") == "postgres" {
|
||||
t.Skip("Skipping on CI and Postgres store")
|
||||
}
|
||||
|
||||
if os.Getenv("NETBIRD_STORE_ENGINE") == "mysql" {
|
||||
t.Skip("Skipping on CI and MySQL store")
|
||||
}
|
||||
}
|
||||
for i := 0; i < 500; i++ {
|
||||
t.Run(fmt.Sprintf("TestRun-%d", i), func(t *testing.T) {
|
||||
|
||||
@@ -19,19 +19,11 @@ import (
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
func GetColumnName(db *gorm.DB, column string) string {
|
||||
if db.Name() == "mysql" {
|
||||
return fmt.Sprintf("`%s`", column)
|
||||
}
|
||||
return column
|
||||
}
|
||||
|
||||
// MigrateFieldFromGobToJSON migrates a column from Gob encoding to JSON encoding.
|
||||
// T is the type of the model that contains the field to be migrated.
|
||||
// S is the type of the field to be migrated.
|
||||
func MigrateFieldFromGobToJSON[T any, S any](ctx context.Context, db *gorm.DB, fieldName string) error {
|
||||
orgColumnName := fieldName
|
||||
oldColumnName := GetColumnName(db, orgColumnName)
|
||||
oldColumnName := fieldName
|
||||
newColumnName := fieldName + "_tmp"
|
||||
|
||||
var model T
|
||||
@@ -86,7 +78,7 @@ func MigrateFieldFromGobToJSON[T any, S any](ctx context.Context, db *gorm.DB, f
|
||||
for _, row := range rows {
|
||||
var field S
|
||||
|
||||
str, ok := row[orgColumnName].(string)
|
||||
str, ok := row[oldColumnName].(string)
|
||||
if !ok {
|
||||
return fmt.Errorf("type assertion failed")
|
||||
}
|
||||
@@ -125,8 +117,7 @@ func MigrateFieldFromGobToJSON[T any, S any](ctx context.Context, db *gorm.DB, f
|
||||
// MigrateNetIPFieldFromBlobToJSON migrates a Net IP column from Blob encoding to JSON encoding.
|
||||
// T is the type of the model that contains the field to be migrated.
|
||||
func MigrateNetIPFieldFromBlobToJSON[T any](ctx context.Context, db *gorm.DB, fieldName string, indexName string) error {
|
||||
orgColumnName := fieldName
|
||||
oldColumnName := GetColumnName(db, orgColumnName)
|
||||
oldColumnName := fieldName
|
||||
newColumnName := fieldName + "_tmp"
|
||||
|
||||
var model T
|
||||
@@ -178,7 +169,7 @@ func MigrateNetIPFieldFromBlobToJSON[T any](ctx context.Context, db *gorm.DB, fi
|
||||
|
||||
for _, row := range rows {
|
||||
var blobValue string
|
||||
if columnValue := row[orgColumnName]; columnValue != nil {
|
||||
if columnValue := row[oldColumnName]; columnValue != nil {
|
||||
value, ok := columnValue.(string)
|
||||
if !ok {
|
||||
return fmt.Errorf("type assertion failed")
|
||||
@@ -225,8 +216,7 @@ func MigrateNetIPFieldFromBlobToJSON[T any](ctx context.Context, db *gorm.DB, fi
|
||||
}
|
||||
|
||||
func MigrateSetupKeyToHashedSetupKey[T any](ctx context.Context, db *gorm.DB) error {
|
||||
orgColumnName := "key"
|
||||
oldColumnName := GetColumnName(db, orgColumnName)
|
||||
oldColumnName := "key"
|
||||
newColumnName := "key_secret"
|
||||
|
||||
var model T
|
||||
@@ -268,7 +258,7 @@ func MigrateSetupKeyToHashedSetupKey[T any](ctx context.Context, db *gorm.DB) er
|
||||
for _, row := range rows {
|
||||
|
||||
var plainKey string
|
||||
if columnValue := row[orgColumnName]; columnValue != nil {
|
||||
if columnValue := row[oldColumnName]; columnValue != nil {
|
||||
value, ok := columnValue.(string)
|
||||
if !ok {
|
||||
return fmt.Errorf("type assertion failed")
|
||||
@@ -394,33 +384,13 @@ func CreateIndexIfNotExists[T any](ctx context.Context, db *gorm.DB, indexName s
|
||||
return fmt.Errorf("failed to parse model schema: %w", err)
|
||||
}
|
||||
tableName := stmt.Schema.Table
|
||||
dialect := db.Name()
|
||||
|
||||
if db.Migrator().HasIndex(&model, indexName) {
|
||||
log.WithContext(ctx).Infof("index %s already exists on table %s", indexName, tableName)
|
||||
return nil
|
||||
}
|
||||
|
||||
var columnClause string
|
||||
if dialect == "mysql" {
|
||||
var withLength []string
|
||||
for _, col := range columns {
|
||||
quotedCol := fmt.Sprintf("`%s`", col)
|
||||
if col == "ip" || col == "dns_label" || col == "key" {
|
||||
withLength = append(withLength, fmt.Sprintf("%s(64)", quotedCol))
|
||||
} else {
|
||||
withLength = append(withLength, quotedCol)
|
||||
}
|
||||
}
|
||||
columnClause = strings.Join(withLength, ", ")
|
||||
} else {
|
||||
columnClause = strings.Join(columns, ", ")
|
||||
}
|
||||
|
||||
createStmt := fmt.Sprintf("CREATE UNIQUE INDEX %s ON %s (%s)", indexName, tableName, columnClause)
|
||||
if dialect == "postgres" || dialect == "sqlite" {
|
||||
createStmt = strings.Replace(createStmt, "CREATE UNIQUE INDEX", "CREATE UNIQUE INDEX IF NOT EXISTS", 1)
|
||||
}
|
||||
createStmt := fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, tableName, strings.Join(columns, ", "))
|
||||
|
||||
log.WithContext(ctx).Infof("executing index creation: %s", createStmt)
|
||||
if err := db.Exec(createStmt).Error; err != nil {
|
||||
@@ -505,14 +475,6 @@ func hasForeignKey(db *gorm.DB, table, column string) bool {
|
||||
AND kcu.table_name = ?
|
||||
AND kcu.column_name = ?
|
||||
`, table, column).Scan(&count)
|
||||
case "mysql":
|
||||
db.Raw(`
|
||||
SELECT COUNT(*) FROM information_schema.key_column_usage
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = ?
|
||||
AND column_name = ?
|
||||
AND referenced_table_name IS NOT NULL
|
||||
`, table, column).Scan(&count)
|
||||
default: // sqlite
|
||||
type fkInfo struct {
|
||||
From string
|
||||
@@ -592,7 +554,7 @@ func RemoveDuplicatePeerKeys(ctx context.Context, db *gorm.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
keyColumn := GetColumnName(db, "key")
|
||||
keyColumn := "key"
|
||||
|
||||
var duplicates []struct {
|
||||
Key string
|
||||
|
||||
@@ -33,15 +33,6 @@ func (agentNetworkSettingsMigration) TableName() string { return "agent_network_
|
||||
// bootstrap always wrote both, so such a row indicates corruption and the
|
||||
// migration fails loudly rather than leaving an empty domain to collide with
|
||||
// the unique index confusingly.
|
||||
//
|
||||
// The transaction is real only on sqlite and postgres, where DDL is
|
||||
// transactional. MySQL implicitly commits around every ALTER TABLE, so there
|
||||
// each step stands alone; what makes an interrupted run resumable on MySQL is
|
||||
// that every step is guarded by the schema state it changes — the entry check
|
||||
// fires while either legacy column remains, the adds skip existing columns,
|
||||
// the backfill and its loud-failure check run only while the legacy cluster
|
||||
// column exists (they provably completed before any drop), and each drop
|
||||
// skips what is already gone.
|
||||
func MigrateAgentNetworkSettingsToDomain(ctx context.Context, db *gorm.DB) error {
|
||||
model := &agentNetworkSettingsMigration{}
|
||||
migrator := db.Migrator()
|
||||
@@ -66,14 +57,9 @@ func MigrateAgentNetworkSettingsToDomain(ctx context.Context, db *gorm.DB) error
|
||||
}
|
||||
|
||||
if hasCluster {
|
||||
concat := "subdomain || '.' || cluster"
|
||||
if tx.Name() == "mysql" {
|
||||
concat = "CONCAT(subdomain, '.', cluster)"
|
||||
}
|
||||
res := tx.Exec(fmt.Sprintf(
|
||||
"UPDATE agent_network_settings SET domain = %s, proxy_address = cluster WHERE (domain IS NULL OR domain = '') AND cluster <> '' AND subdomain <> ''",
|
||||
concat,
|
||||
))
|
||||
res := tx.Exec(
|
||||
"UPDATE agent_network_settings SET domain = subdomain || '.' || cluster, proxy_address = cluster WHERE (domain IS NULL OR domain = '') AND cluster <> '' AND subdomain <> ''",
|
||||
)
|
||||
if res.Error != nil {
|
||||
return fmt.Errorf("backfill agent_network_settings domain: %w", res.Error)
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
@@ -32,17 +31,6 @@ func setupDatabase(t *testing.T) *gorm.DB {
|
||||
var dsn string
|
||||
var cleanup func()
|
||||
switch os.Getenv("NETBIRD_STORE_ENGINE") {
|
||||
case "mysql":
|
||||
cleanup, dsn, err = testutil.CreateMysqlTestContainer()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create MySQL test container: %v", err)
|
||||
}
|
||||
|
||||
if dsn == "" {
|
||||
t.Fatal("MySQL connection string is empty, ensure the test container is running")
|
||||
}
|
||||
|
||||
db, err = gorm.Open(mysql.Open(dsn+"?charset=utf8&parseTime=True&loc=Local"), &gorm.Config{})
|
||||
case "postgres":
|
||||
cleanup, dsn, err = testutil.CreatePostgresTestContainer()
|
||||
if err != nil {
|
||||
@@ -597,9 +585,8 @@ func TestCleanupOrphanedResources_Idempotent(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCleanupOrphanedResources_SkipsWhenForeignKeyExists(t *testing.T) {
|
||||
engine := os.Getenv("NETBIRD_STORE_ENGINE")
|
||||
if engine != "postgres" && engine != "mysql" {
|
||||
t.Skip("FK constraint early-exit test requires postgres or mysql")
|
||||
if os.Getenv("NETBIRD_STORE_ENGINE") != "postgres" {
|
||||
t.Skip("FK constraint early-exit test requires postgres")
|
||||
}
|
||||
|
||||
db := setupDatabase(t)
|
||||
@@ -614,24 +601,12 @@ func TestCleanupOrphanedResources_SkipsWhenForeignKeyExists(t *testing.T) {
|
||||
require.NoError(t, db.Create(&testChildWithFK{ID: "c1", ParentID: "p1"}).Error)
|
||||
require.NoError(t, db.Create(&testChildWithFK{ID: "c2", ParentID: "p2"}).Error)
|
||||
|
||||
switch engine {
|
||||
case "postgres":
|
||||
require.NoError(t, db.Exec("ALTER TABLE test_children DROP CONSTRAINT fk_test_children_parent").Error)
|
||||
require.NoError(t, db.Exec("DELETE FROM test_parents WHERE id = ?", "p2").Error)
|
||||
require.NoError(t, db.Exec(
|
||||
"ALTER TABLE test_children ADD CONSTRAINT fk_test_children_parent "+
|
||||
"FOREIGN KEY (parent_id) REFERENCES test_parents(id) NOT VALID",
|
||||
).Error)
|
||||
case "mysql":
|
||||
require.NoError(t, db.Exec("SET FOREIGN_KEY_CHECKS = 0").Error)
|
||||
require.NoError(t, db.Exec("ALTER TABLE test_children DROP FOREIGN KEY fk_test_children_parent").Error)
|
||||
require.NoError(t, db.Exec("DELETE FROM test_parents WHERE id = ?", "p2").Error)
|
||||
require.NoError(t, db.Exec(
|
||||
"ALTER TABLE test_children ADD CONSTRAINT fk_test_children_parent "+
|
||||
"FOREIGN KEY (parent_id) REFERENCES test_parents(id)",
|
||||
).Error)
|
||||
require.NoError(t, db.Exec("SET FOREIGN_KEY_CHECKS = 1").Error)
|
||||
}
|
||||
require.NoError(t, db.Exec("ALTER TABLE test_children DROP CONSTRAINT fk_test_children_parent").Error)
|
||||
require.NoError(t, db.Exec("DELETE FROM test_parents WHERE id = ?", "p2").Error)
|
||||
require.NoError(t, db.Exec(
|
||||
"ALTER TABLE test_children ADD CONSTRAINT fk_test_children_parent "+
|
||||
"FOREIGN KEY (parent_id) REFERENCES test_parents(id) NOT VALID",
|
||||
).Error)
|
||||
|
||||
err = migration.CleanupOrphanedResources[testChildWithFK, testParent](context.Background(), db, "parent_id")
|
||||
require.NoError(t, err)
|
||||
@@ -717,8 +692,6 @@ func TestFoldCostAggregatesIntoBuckets_SkipsAlreadyMigrated(t *testing.T) {
|
||||
require.NoError(t, db.Migrator().DropTable(&agentNetworkTypes.AgentNetworkUsage{}))
|
||||
|
||||
require.NoError(t, db.AutoMigrate(&agentNetworkTypes.AgentNetworkUsage{}))
|
||||
// Timestamp must be set explicitly: a zero time.Time serialises as
|
||||
// '0000-00-00 00:00:00', which MySQL rejects under strict mode.
|
||||
require.NoError(t, db.Create(&agentNetworkTypes.AgentNetworkUsage{
|
||||
ID: "u1", AccountID: "acct-1", Model: "claude-sonnet-4-6",
|
||||
Timestamp: time.Date(2026, 5, 5, 9, 0, 0, 0, time.UTC),
|
||||
@@ -819,10 +792,9 @@ func TestMigrateAgentNetworkSettingsToDomain_FailsOnUnmigratableRow(t *testing.T
|
||||
assert.Contains(t, err.Error(), "resolve them manually", "the error must tell the operator what to do")
|
||||
}
|
||||
|
||||
// partialAgentNetworkSettings models the one non-atomic state a MySQL run can
|
||||
// be interrupted in: DDL auto-commits there, so a crash between the two legacy
|
||||
// column drops leaves subdomain behind while cluster (and the completed
|
||||
// backfill) are already committed.
|
||||
// partialAgentNetworkSettings models a schema left between the two legacy
|
||||
// column drops: subdomain is still present while cluster is gone and the
|
||||
// backfill has already completed.
|
||||
type partialAgentNetworkSettings struct {
|
||||
AccountID string `gorm:"primaryKey"`
|
||||
Subdomain string
|
||||
@@ -832,8 +804,8 @@ type partialAgentNetworkSettings struct {
|
||||
|
||||
func (partialAgentNetworkSettings) TableName() string { return "agent_network_settings" }
|
||||
|
||||
// TestMigrateAgentNetworkSettingsToDomain_ResumesAfterPartialDrop pins MySQL
|
||||
// resumability: a rerun over the interrupted state must remove the leftover
|
||||
// TestMigrateAgentNetworkSettingsToDomain_ResumesAfterPartialDrop pins
|
||||
// resumability: a rerun over the partial state must remove the leftover
|
||||
// subdomain column without re-running the backfill (the cluster column that
|
||||
// feeds it is gone) and without touching the migrated values.
|
||||
func TestMigrateAgentNetworkSettingsToDomain_ResumesAfterPartialDrop(t *testing.T) {
|
||||
|
||||
@@ -1566,8 +1566,7 @@ func Test_RegisterPeerBySetupKey(t *testing.T) {
|
||||
|
||||
func Test_RegisterPeerRollbackOnFailure(t *testing.T) {
|
||||
engine := os.Getenv("NETBIRD_STORE_ENGINE")
|
||||
if engine == "sqlite" || engine == "mysql" || engine == "" {
|
||||
// we intentionally disabled foreign keys in mysql
|
||||
if engine == "sqlite" || engine == "" {
|
||||
t.Skip("Skipping test because store is not respecting foreign keys")
|
||||
}
|
||||
if runtime.GOOS == "windows" {
|
||||
@@ -2219,10 +2218,6 @@ func Test_IsUniqueConstraintError(t *testing.T) {
|
||||
name: "PostgreSQL uniqueness error",
|
||||
engine: types.PostgresStoreEngine,
|
||||
},
|
||||
{
|
||||
name: "MySQL uniqueness error",
|
||||
engine: types.MysqlStoreEngine,
|
||||
},
|
||||
{
|
||||
name: "SQLite uniqueness error",
|
||||
engine: types.SqliteStoreEngine,
|
||||
|
||||
@@ -33,7 +33,6 @@ import (
|
||||
const (
|
||||
idQueryCondition = "id = ?"
|
||||
keyQueryCondition = "key = ?"
|
||||
mysqlKeyQueryCondition = "`key` = ?"
|
||||
accountAndIDQueryCondition = "account_id = ? and id = ?"
|
||||
accountAndAnyIDQueryCondition = "account_id = ? and (id = ? or public_id = ?)"
|
||||
accountAndPeerIDQueryCondition = "account_id = ? and peer_id = ?"
|
||||
@@ -119,13 +118,6 @@ func (s *SqlStore) pgxPool() *pgxpool.Pool {
|
||||
return s.conn.Pool(s.tx)
|
||||
}
|
||||
|
||||
func GetKeyQueryCondition(s *SqlStore) string {
|
||||
if s.conn.Engine() == db.MysqlStoreEngine {
|
||||
return mysqlKeyQueryCondition
|
||||
}
|
||||
return keyQueryCondition
|
||||
}
|
||||
|
||||
// AcquireGlobalLock acquires global lock across all the accounts and returns a function that releases the lock
|
||||
func (s *SqlStore) AcquireGlobalLock(ctx context.Context) (unlock func()) {
|
||||
log.WithContext(ctx).Tracef("acquiring global lock")
|
||||
@@ -174,15 +166,6 @@ func NewPostgresqlStore(ctx context.Context, dsn string, metrics telemetry.AppMe
|
||||
return newStore(ctx, conn, metrics, skipMigration)
|
||||
}
|
||||
|
||||
// NewMysqlStore creates a new MySQL store.
|
||||
func NewMysqlStore(ctx context.Context, dsn string, metrics telemetry.AppMetrics, skipMigration bool) (*SqlStore, error) {
|
||||
conn, err := db.OpenMysql(ctx, dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newStore(ctx, conn, metrics, skipMigration)
|
||||
}
|
||||
|
||||
// NewSqliteStoreFromFileStore restores a store from FileStore and stores SQLite DB in the file located in datadir.
|
||||
func NewSqliteStoreFromFileStore(ctx context.Context, fileStore *FileStore, dataDir string, metrics telemetry.AppMetrics, skipMigration bool) (*SqlStore, error) {
|
||||
store, err := NewSqliteStore(ctx, dataDir, metrics, skipMigration)
|
||||
@@ -240,11 +223,6 @@ func NewPostgresqlStoreForTests(ctx context.Context, dsn string, metrics telemet
|
||||
return newStore(ctx, conn, metrics, skipMigration)
|
||||
}
|
||||
|
||||
// NewMysqlStoreFromSqlStore restores a store from SqlStore and stores MySQL DB.
|
||||
func NewMysqlStoreFromSqlStore(ctx context.Context, sqliteStore *SqlStore, dsn string, metrics telemetry.AppMetrics) (*SqlStore, error) {
|
||||
return newMysqlStoreFromSqlStore(ctx, sqliteStore, dsn, metrics, false)
|
||||
}
|
||||
|
||||
// seedFromSqliteStore copies the installation ID and the accounts of the
|
||||
// sqlite seed store into a freshly created engine store.
|
||||
func seedFromSqliteStore(ctx context.Context, store, sqliteStore *SqlStore) error {
|
||||
@@ -259,20 +237,6 @@ func seedFromSqliteStore(ctx context.Context, store, sqliteStore *SqlStore) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
func newMysqlStoreFromSqlStore(ctx context.Context, sqliteStore *SqlStore, dsn string, metrics telemetry.AppMetrics, skipMigration bool) (*SqlStore, error) {
|
||||
store, err := NewMysqlStore(ctx, dsn, metrics, skipMigration)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := seedFromSqliteStore(ctx, store, sqliteStore); err != nil {
|
||||
_ = store.Close(ctx)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return store, nil
|
||||
}
|
||||
|
||||
// ExecuteInTransaction runs operation in a transaction. A store that is already
|
||||
// bound to one joins it instead of opening a second, independent transaction.
|
||||
func (s *SqlStore) ExecuteInTransaction(ctx context.Context, operation func(store Store) error) error {
|
||||
|
||||
@@ -1084,10 +1084,8 @@ func (s *SqlStore) SaveAccountSettings(ctx context.Context, accountID string, se
|
||||
return status.Errorf(status.Internal, "failed to save account settings to store")
|
||||
}
|
||||
|
||||
// MySQL reports RowsAffected=0 for no-op updates where values don't change,
|
||||
// unlike SQLite/Postgres which report matched rows. Skip the check since the
|
||||
// caller (UpdateAccountSettings) already verified the account exists via
|
||||
// GetAccountSettings with LockingStrengthUpdate.
|
||||
// No RowsAffected check: the caller (UpdateAccountSettings) already verified
|
||||
// the account exists via GetAccountSettings with LockingStrengthUpdate.
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package store
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -69,10 +68,6 @@ func TestSqlStore_GetGroupsByIDs(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSqlStore_CreateGroup(t *testing.T) {
|
||||
if os.Getenv("CI") == "true" {
|
||||
t.Log("Skipping MySQL test on CI")
|
||||
}
|
||||
t.Setenv("NETBIRD_STORE_ENGINE", string(types.MysqlStoreEngine))
|
||||
store, cleanup, err := NewTestStoreFromSQL(context.Background(), "../testdata/extended-store.sql", t.TempDir())
|
||||
t.Cleanup(cleanup)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -55,7 +55,6 @@ func (s *SqlStore) ListUsers(ctx context.Context) ([]*types.User, error) {
|
||||
}
|
||||
|
||||
// txDeferFKConstraints defers foreign key constraint checks for the duration of the transaction.
|
||||
// MySQL is already handled by s.transaction (SET FOREIGN_KEY_CHECKS = 0).
|
||||
func (s *SqlStore) txDeferFKConstraints(tx *gorm.DB) error {
|
||||
if s.conn.Engine() == types.SqliteStoreEngine {
|
||||
return tx.Exec("PRAGMA defer_foreign_keys = ON").Error
|
||||
|
||||
@@ -375,7 +375,7 @@ func (s *SqlStore) GetAccountByPeerID(ctx context.Context, peerID string) (*type
|
||||
|
||||
func (s *SqlStore) GetAccountByPeerPubKey(ctx context.Context, peerKey string) (*types.Account, error) {
|
||||
var peer nbpeer.Peer
|
||||
result := s.db.Select("account_id").Take(&peer, GetKeyQueryCondition(s), peerKey)
|
||||
result := s.db.Select("account_id").Take(&peer, keyQueryCondition, peerKey)
|
||||
|
||||
if result.Error != nil {
|
||||
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
|
||||
@@ -394,7 +394,7 @@ func (s *SqlStore) GetAccountByPeerPubKey(ctx context.Context, peerKey string) (
|
||||
func (s *SqlStore) GetAccountIDByPeerPubKey(ctx context.Context, peerKey string) (string, error) {
|
||||
var peer nbpeer.Peer
|
||||
var accountID string
|
||||
result := s.db.Model(&peer).Select("account_id").Where(GetKeyQueryCondition(s), peerKey).Take(&accountID)
|
||||
result := s.db.Model(&peer).Select("account_id").Where(keyQueryCondition, peerKey).Take(&accountID)
|
||||
if result.Error != nil {
|
||||
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
|
||||
return "", status.Errorf(status.NotFound, "account not found: index lookup failed")
|
||||
@@ -483,7 +483,7 @@ func (s *SqlStore) GetPeerByPeerPubKey(ctx context.Context, lockStrength Locking
|
||||
}
|
||||
|
||||
var peer nbpeer.Peer
|
||||
result := tx.Take(&peer, GetKeyQueryCondition(s), peerKey)
|
||||
result := tx.Take(&peer, keyQueryCondition, peerKey)
|
||||
|
||||
if result.Error != nil {
|
||||
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
|
||||
@@ -914,7 +914,7 @@ func (s *SqlStore) GetUserIDByPeerKey(ctx context.Context, lockStrength LockingS
|
||||
var userID string
|
||||
result := tx.Model(&nbpeer.Peer{}).
|
||||
Select("user_id").
|
||||
Take(&userID, GetKeyQueryCondition(s), peerKey)
|
||||
Take(&userID, keyQueryCondition, peerKey)
|
||||
|
||||
if result.Error != nil {
|
||||
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
|
||||
@@ -935,7 +935,7 @@ func (s *SqlStore) GetPeerIDByKey(ctx context.Context, lockStrength LockingStren
|
||||
var peerID string
|
||||
result := tx.Model(&nbpeer.Peer{}).
|
||||
Select("id").
|
||||
Where(GetKeyQueryCondition(s), key).
|
||||
Where(keyQueryCondition, key).
|
||||
Limit(1).
|
||||
Scan(&peerID)
|
||||
if result.Error != nil {
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
)
|
||||
|
||||
// TestPgxServiceColumnsMatchGorm guards the Postgres pgx read path against
|
||||
// drifting from the gorm model. The SQLite/MySQL gorm path loads rows by struct,
|
||||
// drifting from the gorm model. The SQLite gorm path loads rows by struct,
|
||||
// so a new column on a model is picked up automatically, but the hand-written
|
||||
// pgx SELECT in sql_store.go must be updated by hand. This test fails when a
|
||||
// gorm column is missing from the pgx column list, which otherwise silently
|
||||
|
||||
@@ -21,8 +21,8 @@ func (s *SqlStore) GetProxyMetrics(ctx context.Context) (ProxyMetrics, error) {
|
||||
var m ProxyMetrics
|
||||
activeCutoff := time.Now().Add(-proxyActiveThreshold)
|
||||
|
||||
// COUNT(DISTINCT ... CASE WHEN ...) is portable across sqlite/postgres
|
||||
// (MySQL too) and keeps the round-trip to one. proxy.StatusConnected
|
||||
// COUNT(DISTINCT ... CASE WHEN ...) is portable across sqlite and postgres
|
||||
// and keeps the round-trip to one. proxy.StatusConnected
|
||||
// is the same string the cluster-capability queries use; the active
|
||||
// window matches the cluster-capability semantics (only proxies
|
||||
// heartbeating within ~2 * heartbeat interval count as connected).
|
||||
@@ -292,7 +292,7 @@ func (s *SqlStore) GetProxyClusters(ctx context.Context, accountID string) ([]pr
|
||||
"cluster_address AS address, "+
|
||||
// COUNT(CASE WHEN ... THEN 1 END) counts only non-NULL — i.e. only
|
||||
// rows that satisfy the predicate — so it works portably across
|
||||
// sqlite/postgres/mysql without dialect-specific FILTER syntax.
|
||||
// sqlite and postgres without dialect-specific FILTER syntax.
|
||||
"COUNT(CASE WHEN status = ? AND last_seen > ? THEN 1 END) AS connected_proxies, "+
|
||||
// MAX(CASE …) > 0 expresses BOOL_OR in a way Postgres tolerates
|
||||
// (Postgres can't MAX a boolean column).
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
|
||||
func (s *SqlStore) GetAccountBySetupKey(ctx context.Context, setupKey string) (*types.Account, error) {
|
||||
var key types.SetupKey
|
||||
result := s.db.Select("account_id").Take(&key, GetKeyQueryCondition(s), setupKey)
|
||||
result := s.db.Select("account_id").Take(&key, keyQueryCondition, setupKey)
|
||||
if result.Error != nil {
|
||||
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
|
||||
return nil, status.NewSetupKeyNotFoundError(setupKey)
|
||||
@@ -99,7 +99,7 @@ func (s *SqlStore) getSetupKeys(ctx context.Context, accountID string) ([]types.
|
||||
|
||||
func (s *SqlStore) GetAccountIDBySetupKey(ctx context.Context, setupKey string) (string, error) {
|
||||
var accountID string
|
||||
result := s.db.Model(&types.SetupKey{}).Select("account_id").Where(GetKeyQueryCondition(s), setupKey).Take(&accountID)
|
||||
result := s.db.Model(&types.SetupKey{}).Select("account_id").Where(keyQueryCondition, setupKey).Take(&accountID)
|
||||
if result.Error != nil {
|
||||
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
|
||||
return "", status.NewSetupKeyNotFoundError(setupKey)
|
||||
@@ -123,7 +123,7 @@ func (s *SqlStore) GetSetupKeyBySecret(ctx context.Context, lockStrength Locking
|
||||
|
||||
var setupKey types.SetupKey
|
||||
result := tx.
|
||||
Take(&setupKey, GetKeyQueryCondition(s), key)
|
||||
Take(&setupKey, keyQueryCondition, key)
|
||||
|
||||
if result.Error != nil {
|
||||
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
|
||||
|
||||
@@ -164,7 +164,6 @@ func TestMigrate(t *testing.T) {
|
||||
|
||||
err = migratePreAuto(context.Background(), store.(*SqlStore).db)
|
||||
require.NoError(t, err, "Migration should not fail on migrated db")
|
||||
|
||||
}
|
||||
|
||||
func newAccount(store Store, id int) error {
|
||||
@@ -352,10 +351,6 @@ func TestSqlStore_DatabaseBlocking(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSqlStore_ExecuteInTransaction_Timeout(t *testing.T) {
|
||||
if os.Getenv("NETBIRD_STORE_ENGINE") == "mysql" {
|
||||
t.Skip("Skipping timeout test for MySQL")
|
||||
}
|
||||
|
||||
t.Setenv("NB_STORE_TRANSACTION_TIMEOUT", "1s")
|
||||
|
||||
store, cleanup, err := NewTestStoreFromSQL(context.Background(), "", t.TempDir())
|
||||
@@ -429,49 +424,6 @@ func TestNewSqliteStore_BusyTimeoutRespectsUserOverride(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSqlStore_ExecuteInTransaction_RestoresForeignKeyChecksOnMysql(t *testing.T) {
|
||||
runTestForAllEngines(t, "", func(t *testing.T, store Store) {
|
||||
sqlStore := store.(*SqlStore)
|
||||
if sqlStore.conn.Engine() != types.MysqlStoreEngine {
|
||||
t.Skip("FOREIGN_KEY_CHECKS is MySQL specific")
|
||||
}
|
||||
sqlDB, err := sqlStore.GetDB().DB()
|
||||
require.NoError(t, err)
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
ctx := context.Background()
|
||||
|
||||
foreignKeyChecks := func() int {
|
||||
var enabled int
|
||||
require.NoError(t, sqlStore.GetDB().Raw("SELECT @@SESSION.foreign_key_checks").Scan(&enabled).Error)
|
||||
return enabled
|
||||
}
|
||||
|
||||
err = store.ExecuteInTransaction(ctx, func(Store) error { return assert.AnError })
|
||||
require.ErrorIs(t, err, assert.AnError)
|
||||
assert.Equal(t, 1, foreignKeyChecks())
|
||||
|
||||
require.Panics(t, func() {
|
||||
_ = store.ExecuteInTransaction(ctx, func(Store) error { panic("boom") })
|
||||
})
|
||||
assert.Equal(t, 1, foreignKeyChecks())
|
||||
|
||||
err = sqlStore.transaction(ctx, func(*gorm.DB) error { return assert.AnError })
|
||||
require.ErrorIs(t, err, assert.AnError)
|
||||
assert.Equal(t, 1, foreignKeyChecks())
|
||||
|
||||
err = store.ExecuteInTransaction(ctx, func(transaction Store) error {
|
||||
bound := transaction.(*SqlStore)
|
||||
require.NoError(t, bound.transaction(ctx, func(*gorm.DB) error { return nil }))
|
||||
var enabled int
|
||||
require.NoError(t, bound.GetDB().Raw("SELECT @@SESSION.foreign_key_checks").Scan(&enabled).Error)
|
||||
assert.Equal(t, 0, enabled, "a savepoint must not re-enable FK checks for the rest of the transaction")
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, foreignKeyChecks())
|
||||
})
|
||||
}
|
||||
|
||||
func TestSqlStore_Transaction_RollsBackOnError(t *testing.T) {
|
||||
runTestForAllEngines(t, "../testdata/extended-store.sql", func(t *testing.T, store Store) {
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -4,7 +4,6 @@ package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
@@ -20,7 +19,6 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
@@ -452,8 +450,6 @@ type AgentNetworkMetrics struct {
|
||||
const (
|
||||
PostgresDsnEnv = "NB_STORE_ENGINE_POSTGRES_DSN"
|
||||
PostgresDsnEnvLegacy = "NETBIRD_STORE_ENGINE_POSTGRES_DSN"
|
||||
mysqlDsnEnv = "NB_STORE_ENGINE_MYSQL_DSN"
|
||||
mysqlDsnEnvLegacy = "NETBIRD_STORE_ENGINE_MYSQL_DSN"
|
||||
)
|
||||
|
||||
// lookupDSNEnv checks the NB_ env var first, then falls back to the legacy NETBIRD_ env var.
|
||||
@@ -464,7 +460,7 @@ func lookupDSNEnv(nbKey, legacyKey string) (string, bool) {
|
||||
return os.LookupEnv(legacyKey)
|
||||
}
|
||||
|
||||
var supportedEngines = []types.Engine{types.SqliteStoreEngine, types.PostgresStoreEngine, types.MysqlStoreEngine}
|
||||
var supportedEngines = []types.Engine{types.SqliteStoreEngine, types.PostgresStoreEngine}
|
||||
|
||||
func getStoreEngineFromEnv() types.Engine {
|
||||
// NETBIRD_STORE_ENGINE supposed to be used in tests. Otherwise, rely on the config file.
|
||||
@@ -474,7 +470,7 @@ func getStoreEngineFromEnv() types.Engine {
|
||||
}
|
||||
|
||||
value := types.Engine(strings.ToLower(kind))
|
||||
if slices.Contains(supportedEngines, value) {
|
||||
if slices.Contains(supportedEngines, value) || value == types.MysqlStoreEngine {
|
||||
return value
|
||||
}
|
||||
|
||||
@@ -540,12 +536,8 @@ func OpenConn(ctx context.Context, kind types.Engine, dataDir string) (*db.Conn,
|
||||
}
|
||||
return db.OpenPostgres(ctx, dsn, db.DefaultPoolConfig)
|
||||
case types.MysqlStoreEngine:
|
||||
log.WithContext(ctx).Info("using MySQL store engine")
|
||||
dsn, ok := lookupDSNEnv(mysqlDsnEnv, mysqlDsnEnvLegacy)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s is not set", mysqlDsnEnv)
|
||||
}
|
||||
return db.OpenMysql(ctx, dsn)
|
||||
return nil, fmt.Errorf("the MySQL store engine is no longer supported. Please refer to the documentation for " +
|
||||
"migrating to PostgreSQL: https://docs.netbird.io/selfhosted/migration/mysql-to-postgresql")
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported kind of store: %s", kind)
|
||||
}
|
||||
@@ -725,7 +717,7 @@ func NewTestStoreFromSQL(ctx context.Context, filename string, dataDir string) (
|
||||
}
|
||||
|
||||
if filename != "" {
|
||||
err = LoadSQL(conn.DB(nil), filename, translaterForSql(kind))
|
||||
err = LoadSQL(conn.DB(nil), filename)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, nil, fmt.Errorf("failed to load SQL file: %v", err)
|
||||
@@ -761,6 +753,7 @@ func NewTestStoreFromSQL(ctx context.Context, filename string, dataDir string) (
|
||||
return nil, nil, fmt.Errorf("failed to create test store after %d attempts: %v", maxRetries, err)
|
||||
}
|
||||
|
||||
// NewTestStore creates a test store and optionally loads SQL after migrating its schema.
|
||||
func NewTestStore(ctx context.Context, filename string, dataDir string) (Store, func(), error) {
|
||||
kind := getStoreEngineFromEnv()
|
||||
if kind == "" {
|
||||
@@ -786,7 +779,7 @@ func NewTestStore(ctx context.Context, filename string, dataDir string) (Store,
|
||||
}
|
||||
|
||||
if filename != "" {
|
||||
err = LoadSQL(sqlStore.db, filename, translaterForSql(kind))
|
||||
err = LoadSQL(sqlStore.db, filename)
|
||||
if err != nil {
|
||||
_ = sqlStore.Close(ctx)
|
||||
return nil, nil, fmt.Errorf("failed to load SQL file: %v", err)
|
||||
@@ -802,18 +795,6 @@ func NewTestStore(ctx context.Context, filename string, dataDir string) (Store,
|
||||
return sqlStore, cleanup, nil
|
||||
}
|
||||
|
||||
// use to translate double-quote escaped reserved names (ansi)
|
||||
// to mysql tick-escaped ones
|
||||
func translaterForSql(kind types.Engine) *strings.Replacer {
|
||||
if kind != types.MysqlStoreEngine {
|
||||
return strings.NewReplacer()
|
||||
}
|
||||
return strings.NewReplacer(
|
||||
"\"key\"", "`key`",
|
||||
"\"groups\"", "`groups`",
|
||||
)
|
||||
}
|
||||
|
||||
func addAllGroupToAccount(ctx context.Context, store Store) error {
|
||||
allAccounts := store.GetAllAccounts(ctx)
|
||||
for _, account := range allAccounts {
|
||||
@@ -843,9 +824,7 @@ func getSqlStoreEngine(ctx context.Context, sqliteStore *SqlStore, kind types.En
|
||||
var err error
|
||||
switch kind {
|
||||
case types.PostgresStoreEngine:
|
||||
store, cleanup, err = newReusedPostgresStore(ctx, sqliteStore, kind)
|
||||
case types.MysqlStoreEngine:
|
||||
store, cleanup, err = newReusedMysqlStore(ctx, sqliteStore, kind)
|
||||
store, cleanup, err = newReusedPostgresStore(ctx, sqliteStore)
|
||||
default:
|
||||
cleanup = func() {
|
||||
// sqlite doesn't need to be cleaned up
|
||||
@@ -877,7 +856,7 @@ func getSqlStoreEngine(ctx context.Context, sqliteStore *SqlStore, kind types.En
|
||||
return store, closeConnection, nil
|
||||
}
|
||||
|
||||
func newReusedPostgresStore(ctx context.Context, sqliteStore *SqlStore, kind types.Engine) (*SqlStore, func(), error) {
|
||||
func newReusedPostgresStore(ctx context.Context, sqliteStore *SqlStore) (*SqlStore, func(), error) {
|
||||
dsn, ok := lookupDSNEnv(PostgresDsnEnv, PostgresDsnEnvLegacy)
|
||||
if !ok || dsn == "" {
|
||||
var err error
|
||||
@@ -891,7 +870,7 @@ func newReusedPostgresStore(ctx context.Context, sqliteStore *SqlStore, kind typ
|
||||
return nil, nil, fmt.Errorf("%s is not set", PostgresDsnEnv)
|
||||
}
|
||||
|
||||
db, err := openDBWithRetry(dsn, kind, 5)
|
||||
db, err := openDBWithRetry(dsn, 5)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to open postgres connection: %v", err)
|
||||
}
|
||||
@@ -902,7 +881,7 @@ func newReusedPostgresStore(ctx context.Context, sqliteStore *SqlStore, kind typ
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
dsn, cleanup, err := createRandomDB(dsn, db, kind, template)
|
||||
dsn, cleanup, err := createRandomDB(dsn, db, template)
|
||||
|
||||
closeGormDB(db)
|
||||
|
||||
@@ -928,74 +907,10 @@ func newReusedPostgresStore(ctx context.Context, sqliteStore *SqlStore, kind typ
|
||||
return store, cleanup, nil
|
||||
}
|
||||
|
||||
func newReusedMysqlStore(ctx context.Context, sqliteStore *SqlStore, kind types.Engine) (*SqlStore, func(), error) {
|
||||
dsn, ok := lookupDSNEnv(mysqlDsnEnv, mysqlDsnEnvLegacy)
|
||||
if !ok || dsn == "" {
|
||||
var err error
|
||||
_, dsn, err = testutil.CreateMysqlTestContainer()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if dsn == "" {
|
||||
return nil, nil, fmt.Errorf("%s is not set", mysqlDsnEnv)
|
||||
}
|
||||
|
||||
db, err := openDBWithRetry(dsn, kind, 5)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to open mysql connection: %v", err)
|
||||
}
|
||||
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to get underlying sql.DB: %v", err)
|
||||
}
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
sqlDB.SetMaxIdleConns(1)
|
||||
|
||||
tableDDL, err := mysqlSchemaTemplate(ctx, dsn, db)
|
||||
if err != nil {
|
||||
sqlDB.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
dsn, cleanup, err := createRandomDB(dsn, db, kind, "")
|
||||
|
||||
sqlDB.Close()
|
||||
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if err := cloneMysqlSchema(ctx, dsn, tableDDL); err != nil {
|
||||
cleanup()
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
var store *SqlStore
|
||||
if sqliteStore != nil {
|
||||
store, err = newMysqlStoreFromSqlStore(ctx, sqliteStore, dsn, nil, true)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return nil, nil, err
|
||||
}
|
||||
} else {
|
||||
store, err = NewMysqlStore(ctx, dsn, nil, false)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return store, cleanup, nil
|
||||
}
|
||||
|
||||
// schemaTemplates remembers, per engine and server, a database that went
|
||||
// through the full migration once in this process. Every later test database
|
||||
// is cloned from it, so a test pays for CREATE DATABASE and a schema copy
|
||||
// instead of the 40-table AutoMigrate plus every pre and post migration, which
|
||||
// is what made each MySQL test store cost well over a second in CI.
|
||||
// instead of the 40-table AutoMigrate plus every pre and post migration.
|
||||
var (
|
||||
schemaTemplatesMu sync.Mutex
|
||||
schemaTemplates = map[string]*schemaTemplate{}
|
||||
@@ -1003,10 +918,6 @@ var (
|
||||
|
||||
type schemaTemplate struct {
|
||||
dbName string
|
||||
// tableDDL holds the CREATE TABLE statements of the template. MySQL has no
|
||||
// server-side database template, so the schema is replayed statement by
|
||||
// statement into each test database.
|
||||
tableDDL []string
|
||||
}
|
||||
|
||||
func schemaTemplateKey(engine types.Engine, dsn string) string {
|
||||
@@ -1046,112 +957,6 @@ func postgresSchemaTemplate(ctx context.Context, baseDSN string, admin *gorm.DB)
|
||||
return name, nil
|
||||
}
|
||||
|
||||
// mysqlSchemaTemplate returns the CREATE TABLE statements of a fully migrated
|
||||
// database, migrating one on first use.
|
||||
func mysqlSchemaTemplate(ctx context.Context, baseDSN string, admin *gorm.DB) ([]string, error) {
|
||||
schemaTemplatesMu.Lock()
|
||||
defer schemaTemplatesMu.Unlock()
|
||||
|
||||
key := schemaTemplateKey(types.MysqlStoreEngine, baseDSN)
|
||||
if tpl, ok := schemaTemplates[key]; ok {
|
||||
return tpl.tableDDL, nil
|
||||
}
|
||||
|
||||
name := newTestDBName("test_template")
|
||||
if err := admin.Exec(fmt.Sprintf("CREATE DATABASE %s", name)).Error; err != nil {
|
||||
return nil, fmt.Errorf("create mysql template database: %w", err)
|
||||
}
|
||||
|
||||
tplStore, err := NewMysqlStore(ctx, replaceDBName(baseDSN, name), nil, false)
|
||||
if err != nil {
|
||||
dropDatabase(admin, name)
|
||||
return nil, fmt.Errorf("migrate mysql template database: %w", err)
|
||||
}
|
||||
tableDDL, err := mysqlTableDDL(ctx, tplStore.db, name)
|
||||
tplStore.Close(ctx)
|
||||
if err != nil {
|
||||
dropDatabase(admin, name)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
schemaTemplates[key] = &schemaTemplate{dbName: name, tableDDL: tableDDL}
|
||||
return tableDDL, nil
|
||||
}
|
||||
|
||||
func mysqlTableDDL(ctx context.Context, db *gorm.DB, dbName string) ([]string, error) {
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tables, err := mysqlTableNames(ctx, sqlDB, dbName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tableDDL := make([]string, 0, len(tables))
|
||||
for _, table := range tables {
|
||||
var name, createStmt string
|
||||
row := sqlDB.QueryRowContext(ctx, fmt.Sprintf("SHOW CREATE TABLE %s.%s", dbName, table))
|
||||
if err := row.Scan(&name, &createStmt); err != nil {
|
||||
return nil, fmt.Errorf("read create statement of %s: %w", table, err)
|
||||
}
|
||||
tableDDL = append(tableDDL, createStmt)
|
||||
}
|
||||
return tableDDL, nil
|
||||
}
|
||||
|
||||
func mysqlTableNames(ctx context.Context, sqlDB *sql.DB, dbName string) ([]string, error) {
|
||||
rows, err := sqlDB.QueryContext(ctx, fmt.Sprintf("SHOW TABLES FROM %s", dbName))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list template tables: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var tables []string
|
||||
for rows.Next() {
|
||||
var table string
|
||||
if err := rows.Scan(&table); err != nil {
|
||||
return nil, fmt.Errorf("scan template table name: %w", err)
|
||||
}
|
||||
tables = append(tables, table)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("list template tables: %w", err)
|
||||
}
|
||||
return tables, nil
|
||||
}
|
||||
|
||||
// cloneMysqlSchema replays the template's CREATE TABLE statements into the
|
||||
// database the DSN points at.
|
||||
func cloneMysqlSchema(ctx context.Context, dsn string, tableDDL []string) error {
|
||||
gormDB, err := gorm.Open(mysql.Open(db.MysqlDSN(dsn)), db.GormConfig())
|
||||
if err != nil {
|
||||
return fmt.Errorf("connect to test database: %w", err)
|
||||
}
|
||||
sqlDB, err := gormDB.DB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer sqlDB.Close()
|
||||
|
||||
// The statements come out of SHOW TABLES in name order, not dependency
|
||||
// order, and their foreign keys reference tables of the session's default
|
||||
// database. Pin a single connection so the session setting below covers
|
||||
// every statement, and connect straight to the new database so unqualified
|
||||
// references land there.
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
if _, err := sqlDB.ExecContext(ctx, "SET FOREIGN_KEY_CHECKS = 0"); err != nil {
|
||||
return fmt.Errorf("disable foreign key checks: %w", err)
|
||||
}
|
||||
for _, stmt := range tableDDL {
|
||||
if _, err := sqlDB.ExecContext(ctx, stmt); err != nil {
|
||||
return fmt.Errorf("replay table definition: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// dropDatabase removes a template that never became usable, so a failed setup
|
||||
// does not leave it behind on a shared server. The server may still be tearing
|
||||
// down the sessions the failed migration held, so the drop retries while
|
||||
@@ -1168,18 +973,12 @@ func closeGormDB(db *gorm.DB) {
|
||||
}
|
||||
}
|
||||
|
||||
func openDBWithRetry(dsn string, engine types.Engine, maxRetries int) (*gorm.DB, error) {
|
||||
func openDBWithRetry(dsn string, maxRetries int) (*gorm.DB, error) {
|
||||
var gormDB *gorm.DB
|
||||
var err error
|
||||
|
||||
for i := range maxRetries {
|
||||
switch engine {
|
||||
case types.PostgresStoreEngine:
|
||||
gormDB, err = gorm.Open(postgres.Open(dsn), &gorm.Config{})
|
||||
case types.MysqlStoreEngine:
|
||||
gormDB, err = gorm.Open(mysql.Open(db.MysqlDSN(dsn)), &gorm.Config{})
|
||||
}
|
||||
|
||||
gormDB, err = gorm.Open(postgres.Open(dsn), &gorm.Config{})
|
||||
if err == nil {
|
||||
return gormDB, nil
|
||||
}
|
||||
@@ -1193,13 +992,13 @@ func openDBWithRetry(dsn string, engine types.Engine, maxRetries int) (*gorm.DB,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// createRandomDB creates a uniquely named database for one test. On postgres a
|
||||
// non-empty template is copied server-side with CREATE DATABASE ... TEMPLATE.
|
||||
func createRandomDB(dsn string, admin *gorm.DB, engine types.Engine, template string) (string, func(), error) {
|
||||
// createRandomDB creates a uniquely named database for one test. A non-empty
|
||||
// template is copied server-side with CREATE DATABASE ... TEMPLATE.
|
||||
func createRandomDB(dsn string, admin *gorm.DB, template string) (string, func(), error) {
|
||||
dbName := newTestDBName("test_db")
|
||||
|
||||
createStmt := fmt.Sprintf("CREATE DATABASE %s", dbName)
|
||||
if template != "" && engine == types.PostgresStoreEngine {
|
||||
if template != "" {
|
||||
createStmt = fmt.Sprintf("CREATE DATABASE %s TEMPLATE %s", dbName, template)
|
||||
}
|
||||
if err := execWithTemplateRetry(admin, createStmt); err != nil {
|
||||
@@ -1209,59 +1008,28 @@ func createRandomDB(dsn string, admin *gorm.DB, engine types.Engine, template st
|
||||
originalDSN := dsn
|
||||
|
||||
cleanup := func() {
|
||||
var dropDB *gorm.DB
|
||||
var err error
|
||||
|
||||
switch engine {
|
||||
case types.PostgresStoreEngine:
|
||||
dropDB, err = gorm.Open(postgres.Open(originalDSN), &gorm.Config{
|
||||
SkipDefaultTransaction: true,
|
||||
PrepareStmt: false,
|
||||
Logger: logger.Default.LogMode(logger.Info),
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("failed to connect for dropping database %s: %v", dbName, err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if sqlDB, _ := dropDB.DB(); sqlDB != nil {
|
||||
sqlDB.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
dropDB, err := gorm.Open(postgres.Open(originalDSN), &gorm.Config{
|
||||
SkipDefaultTransaction: true,
|
||||
PrepareStmt: false,
|
||||
Logger: logger.Default.LogMode(logger.Info),
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("failed to connect for dropping database %s: %v", dbName, err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if sqlDB, _ := dropDB.DB(); sqlDB != nil {
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
sqlDB.SetMaxIdleConns(0)
|
||||
sqlDB.SetConnMaxLifetime(time.Second)
|
||||
sqlDB.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
err = dropDB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s WITH (FORCE)", dbName)).Error
|
||||
|
||||
case types.MysqlStoreEngine:
|
||||
dropDB, err = gorm.Open(mysql.Open(db.MysqlDSN(originalDSN)), &gorm.Config{
|
||||
SkipDefaultTransaction: true,
|
||||
PrepareStmt: false,
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("failed to connect for dropping database %s: %v", dbName, err)
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if sqlDB, _ := dropDB.DB(); sqlDB != nil {
|
||||
sqlDB.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
if sqlDB, _ := dropDB.DB(); sqlDB != nil {
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
sqlDB.SetMaxIdleConns(0)
|
||||
sqlDB.SetConnMaxLifetime(time.Second)
|
||||
}
|
||||
|
||||
err = dropDB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", dbName)).Error
|
||||
if sqlDB, _ := dropDB.DB(); sqlDB != nil {
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
sqlDB.SetMaxIdleConns(0)
|
||||
sqlDB.SetConnMaxLifetime(time.Second)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
if err := dropDB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s WITH (FORCE)", dbName)).Error; err != nil {
|
||||
log.Errorf("failed to drop database %s: %v", dbName, err)
|
||||
}
|
||||
}
|
||||
@@ -1288,7 +1056,8 @@ func replaceDBName(dsn, newDBName string) string {
|
||||
return re.ReplaceAllString(dsn, `${pre}`+newDBName+`${post}`)
|
||||
}
|
||||
|
||||
func LoadSQL(db *gorm.DB, filepath string, translator *strings.Replacer) error {
|
||||
// LoadSQL executes the SQL statements from the given file.
|
||||
func LoadSQL(db *gorm.DB, filepath string) error {
|
||||
sqlContent, err := os.ReadFile(filepath)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1297,7 +1066,7 @@ func LoadSQL(db *gorm.DB, filepath string, translator *strings.Replacer) error {
|
||||
queries := strings.Split(string(sqlContent), ";")
|
||||
|
||||
for _, query := range queries {
|
||||
query = translator.Replace(strings.TrimSpace(query))
|
||||
query = strings.TrimSpace(query)
|
||||
if query != "" {
|
||||
err := db.Exec(query).Error
|
||||
if err != nil {
|
||||
|
||||
@@ -8,74 +8,12 @@ import (
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/testcontainers/testcontainers-go"
|
||||
"github.com/testcontainers/testcontainers-go/modules/mysql"
|
||||
"github.com/testcontainers/testcontainers-go/modules/postgres"
|
||||
testcontainersredis "github.com/testcontainers/testcontainers-go/modules/redis"
|
||||
"github.com/testcontainers/testcontainers-go/wait"
|
||||
)
|
||||
|
||||
var (
|
||||
pgContainer *postgres.PostgresContainer
|
||||
mysqlContainer *mysql.MySQLContainer
|
||||
)
|
||||
|
||||
// CreateMysqlTestContainer creates a new MySQL container for testing.
|
||||
func CreateMysqlTestContainer() (func(), string, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
if mysqlContainer != nil {
|
||||
connStr, err := mysqlContainer.ConnectionString(ctx)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return noOpCleanup, connStr, nil
|
||||
}
|
||||
|
||||
var err error
|
||||
mysqlContainer, err = mysql.Run(ctx,
|
||||
"mlsmaycon/warmed-mysql:8",
|
||||
mysql.WithDatabase("testing"),
|
||||
mysql.WithUsername("root"),
|
||||
mysql.WithPassword("testing"),
|
||||
// Every test creates and drops a database with about 40 tables, so with
|
||||
// the server defaults the run is dominated by durability work: each
|
||||
// CREATE TABLE fsyncs the redo log, the binary log and the doublewrite
|
||||
// buffer. None of it protects anything in a container that is discarded
|
||||
// after the run. Tables stay in per-table files on purpose: in the shared
|
||||
// system tablespace the cost of every CREATE and DROP grew with the number
|
||||
// of databases the run had already created.
|
||||
testcontainers.WithCmd("mysqld",
|
||||
"--innodb-flush-log-at-trx-commit=0",
|
||||
"--innodb-doublewrite=OFF",
|
||||
"--skip-log-bin",
|
||||
),
|
||||
testcontainers.WithWaitStrategy(
|
||||
wait.ForLog("/usr/sbin/mysqld: ready for connections").
|
||||
WithOccurrence(1).WithStartupTimeout(15*time.Second).WithPollInterval(100*time.Millisecond),
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
if mysqlContainer != nil {
|
||||
timeoutCtx, cancelFunc := context.WithTimeout(ctx, 1*time.Second)
|
||||
defer cancelFunc()
|
||||
if err = mysqlContainer.Terminate(timeoutCtx); err != nil {
|
||||
log.WithContext(ctx).Warnf("failed to stop mysql container %s: %s", mysqlContainer.GetContainerID(), err)
|
||||
}
|
||||
mysqlContainer = nil // reset the container to allow recreation
|
||||
}
|
||||
}
|
||||
|
||||
talksConn, err := mysqlContainer.ConnectionString(ctx)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
return cleanup, talksConn, nil
|
||||
}
|
||||
var pgContainer *postgres.PostgresContainer
|
||||
|
||||
// CreatePostgresTestContainer creates a new PostgreSQL container for testing.
|
||||
func CreatePostgresTestContainer() (func(), string, error) {
|
||||
|
||||
@@ -8,12 +8,6 @@ func CreatePostgresTestContainer() (func(), string, error) {
|
||||
}, "", nil
|
||||
}
|
||||
|
||||
func CreateMysqlTestContainer() (func(), string, error) {
|
||||
return func() {
|
||||
// Empty function for MySQL
|
||||
}, "", nil
|
||||
}
|
||||
|
||||
func CreateRedisTestContainer() (func(), string, error) {
|
||||
return func() {
|
||||
// Empty function for Redis
|
||||
|
||||
@@ -8,5 +8,5 @@ const (
|
||||
PostgresStoreEngine = db.PostgresStoreEngine
|
||||
FileStoreEngine Engine = "jsonfile"
|
||||
SqliteStoreEngine = db.SqliteStoreEngine
|
||||
MysqlStoreEngine = db.MysqlStoreEngine
|
||||
MysqlStoreEngine Engine = "mysql"
|
||||
)
|
||||
|
||||
@@ -168,7 +168,7 @@ func TestAccounts_Integration_Update(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// Account deletion on MySQL and PostgreSQL databases causes unknown errors
|
||||
// Account deletion on PostgreSQL databases causes unknown errors
|
||||
// func TestAccounts_Integration_Delete(t *testing.T) {
|
||||
// withBlackBoxServer(t, func(c *rest.Client) {
|
||||
// accounts, err := c.Accounts.List(context.Background())
|
||||
|
||||
@@ -1,661 +0,0 @@
|
||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
Version 3, 19 November 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU Affero General Public License is a free, copyleft license for
|
||||
software and other kinds of works, specifically designed to ensure
|
||||
cooperation with the community in the case of network server software.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
our General Public Licenses are intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
Developers that use our General Public Licenses protect your rights
|
||||
with two steps: (1) assert copyright on the software, and (2) offer
|
||||
you this License which gives you legal permission to copy, distribute
|
||||
and/or modify the software.
|
||||
|
||||
A secondary benefit of defending all users' freedom is that
|
||||
improvements made in alternate versions of the program, if they
|
||||
receive widespread use, become available for other developers to
|
||||
incorporate. Many developers of free software are heartened and
|
||||
encouraged by the resulting cooperation. However, in the case of
|
||||
software used on network servers, this result may fail to come about.
|
||||
The GNU General Public License permits making a modified version and
|
||||
letting the public access it on a server without ever releasing its
|
||||
source code to the public.
|
||||
|
||||
The GNU Affero General Public License is designed specifically to
|
||||
ensure that, in such cases, the modified source code becomes available
|
||||
to the community. It requires the operator of a network server to
|
||||
provide the source code of the modified version running there to the
|
||||
users of that server. Therefore, public use of a modified version, on
|
||||
a publicly accessible server, gives the public access to the source
|
||||
code of the modified version.
|
||||
|
||||
An older license, called the Affero General Public License and
|
||||
published by Affero, was designed to accomplish similar goals. This is
|
||||
a different license, not a version of the Affero GPL, but Affero has
|
||||
released a new version of the Affero GPL which permits relicensing under
|
||||
this license.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU Affero General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Remote Network Interaction; Use with the GNU General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, if you modify the
|
||||
Program, your modified version must prominently offer all users
|
||||
interacting with it remotely through a computer network (if your version
|
||||
supports such interaction) an opportunity to receive the Corresponding
|
||||
Source of your version by providing access to the Corresponding Source
|
||||
from a network server at no charge, through some standard or customary
|
||||
means of facilitating copying of software. This Corresponding Source
|
||||
shall include the Corresponding Source for any work covered by version 3
|
||||
of the GNU General Public License that is incorporated pursuant to the
|
||||
following paragraph.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the work with which it is combined will remain governed by version
|
||||
3 of the GNU General Public License.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU Affero General Public License from time to time. Such new versions
|
||||
will be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU Affero General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU Affero General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU Affero General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If your software can interact with users remotely through a computer
|
||||
network, you should also make sure that it provides a way for users to
|
||||
get its source. For example, if your program is a web application, its
|
||||
interface could display a "Source" link that leads users to an archive
|
||||
of the code. There are many ways you could offer source, and different
|
||||
solutions will be better for different programs; see section 13 for the
|
||||
specific requirements.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU AGPL, see
|
||||
<https://www.gnu.org/licenses/>.
|
||||
@@ -1,357 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"math"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
type dialect struct {
|
||||
quote func(string) string
|
||||
tablesQuery string
|
||||
columnsQuery string
|
||||
}
|
||||
|
||||
var mysqlDialect = dialect{
|
||||
quote: func(name string) string { return "`" + name + "`" },
|
||||
tablesQuery: `SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() AND table_type = 'BASE TABLE'`,
|
||||
columnsQuery: `SELECT column_name FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = ?`,
|
||||
}
|
||||
|
||||
var sqliteDialect = dialect{
|
||||
quote: func(name string) string { return `"` + name + `"` },
|
||||
tablesQuery: `SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'`,
|
||||
columnsQuery: `SELECT name FROM pragma_table_info(?)`,
|
||||
}
|
||||
|
||||
type column struct {
|
||||
name string
|
||||
pgType string
|
||||
}
|
||||
|
||||
// copyStore copies the tables the source shares with the target schema;
|
||||
// source-only tables are legacy and skipped.
|
||||
func copyStore(ctx context.Context, src *sql.DB, d dialect, tx pgx.Tx, skip map[string]bool) (int, int64, error) {
|
||||
srcTables, err := queryStrings(ctx, src, d.tablesQuery)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("list source tables: %w", err)
|
||||
}
|
||||
dstTables, err := pgStrings(ctx, tx, `SELECT table_name FROM information_schema.tables WHERE table_schema = current_schema() AND table_type = 'BASE TABLE'`)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("list target tables: %w", err)
|
||||
}
|
||||
|
||||
var tables []string
|
||||
for _, t := range srcTables {
|
||||
if !skip[t] && slices.Contains(dstTables, t) {
|
||||
tables = append(tables, t)
|
||||
}
|
||||
}
|
||||
if len(tables) == 0 {
|
||||
return 0, 0, fmt.Errorf("no NetBird tables found in the source")
|
||||
}
|
||||
|
||||
ordered, err := parentsFirst(ctx, tx, tables)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
var total int64
|
||||
for _, table := range ordered {
|
||||
n, err := copyTable(ctx, src, d, tx, table)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("table %s: %w", table, err)
|
||||
}
|
||||
total += n
|
||||
}
|
||||
|
||||
if err := resetSequences(ctx, tx, tables); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return len(tables), total, nil
|
||||
}
|
||||
|
||||
func copyTable(ctx context.Context, src *sql.DB, d dialect, tx pgx.Tx, table string) (int64, error) {
|
||||
cols, err := targetColumns(ctx, tx, table)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
srcCols, err := queryStrings(ctx, src, d.columnsQuery, table)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("list source columns: %w", err)
|
||||
}
|
||||
|
||||
names := make([]string, len(cols))
|
||||
quoted := make([]string, len(cols))
|
||||
var missing []string
|
||||
for i, c := range cols {
|
||||
names[i] = c.name
|
||||
quoted[i] = d.quote(c.name)
|
||||
if !slices.Contains(srcCols, c.name) {
|
||||
missing = append(missing, c.name)
|
||||
}
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return 0, fmt.Errorf("source lacks columns %v; start the management server once with this release before migrating", missing)
|
||||
}
|
||||
|
||||
var hasRows bool
|
||||
if err := tx.QueryRow(ctx, fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM %s)", pgx.Identifier{table}.Sanitize())).Scan(&hasRows); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if hasRows {
|
||||
return 0, fmt.Errorf("target already holds data")
|
||||
}
|
||||
|
||||
rows, err := src.QueryContext(ctx, fmt.Sprintf("SELECT %s FROM %s", strings.Join(quoted, ", "), d.quote(table)))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("read source: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return tx.CopyFrom(ctx, pgx.Identifier{table}, names, newRowSource(rows, cols))
|
||||
}
|
||||
|
||||
func targetColumns(ctx context.Context, tx pgx.Tx, table string) ([]column, error) {
|
||||
rows, err := tx.Query(ctx, `SELECT column_name, data_type FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = $1 ORDER BY ordinal_position`, table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pgx.CollectRows(rows, func(row pgx.CollectableRow) (column, error) {
|
||||
var c column
|
||||
err := row.Scan(&c.name, &c.pgType)
|
||||
return c, err
|
||||
})
|
||||
}
|
||||
|
||||
func parentsFirst(ctx context.Context, tx pgx.Tx, tables []string) ([]string, error) {
|
||||
rows, err := tx.Query(ctx, `SELECT child.relname, parent.relname FROM pg_constraint c
|
||||
JOIN pg_class child ON child.oid = c.conrelid
|
||||
JOIN pg_class parent ON parent.oid = c.confrelid
|
||||
JOIN pg_namespace n ON n.oid = child.relnamespace
|
||||
WHERE c.contype = 'f' AND n.nspname = current_schema()`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read foreign keys: %w", err)
|
||||
}
|
||||
parents := map[string][]string{}
|
||||
var child, parent string
|
||||
_, err = pgx.ForEachRow(rows, []any{&child, &parent}, func() error {
|
||||
if child != parent {
|
||||
parents[child] = append(parents[child], parent)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read foreign keys: %w", err)
|
||||
}
|
||||
|
||||
pending := slices.Sorted(slices.Values(tables))
|
||||
done := make(map[string]bool, len(tables))
|
||||
ordered := make([]string, 0, len(tables))
|
||||
for len(pending) > 0 {
|
||||
var next []string
|
||||
for _, t := range pending {
|
||||
if ready(parents[t], tables, done) {
|
||||
ordered = append(ordered, t)
|
||||
done[t] = true
|
||||
} else {
|
||||
next = append(next, t)
|
||||
}
|
||||
}
|
||||
if len(next) == len(pending) {
|
||||
return nil, fmt.Errorf("foreign keys form a cycle between %v", next)
|
||||
}
|
||||
pending = next
|
||||
}
|
||||
return ordered, nil
|
||||
}
|
||||
|
||||
func ready(parents, tables []string, done map[string]bool) bool {
|
||||
for _, p := range parents {
|
||||
if slices.Contains(tables, p) && !done[p] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// resetSequences moves serial sequences past the ids COPY inserted explicitly.
|
||||
func resetSequences(ctx context.Context, tx pgx.Tx, tables []string) error {
|
||||
rows, err := tx.Query(ctx, `SELECT table_name, column_name, pg_get_serial_sequence(quote_ident(table_name), column_name)
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = ANY($1)
|
||||
AND pg_get_serial_sequence(quote_ident(table_name), column_name) IS NOT NULL`, tables)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list sequences: %w", err)
|
||||
}
|
||||
type sequence struct{ table, column, name string }
|
||||
seqs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (sequence, error) {
|
||||
var s sequence
|
||||
err := row.Scan(&s.table, &s.column, &s.name)
|
||||
return s, err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list sequences: %w", err)
|
||||
}
|
||||
|
||||
for _, s := range seqs {
|
||||
q := fmt.Sprintf("SELECT setval('%s', COALESCE(MAX(%s), 0) + 1, false) FROM %s",
|
||||
strings.ReplaceAll(s.name, "'", "''"), pgx.Identifier{s.column}.Sanitize(), pgx.Identifier{s.table}.Sanitize())
|
||||
if _, err := tx.Exec(ctx, q); err != nil {
|
||||
return fmt.Errorf("reset sequence %s: %w", s.name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type rowSource struct {
|
||||
rows *sql.Rows
|
||||
cols []column
|
||||
vals []any
|
||||
ptrs []any
|
||||
}
|
||||
|
||||
func newRowSource(rows *sql.Rows, cols []column) *rowSource {
|
||||
r := &rowSource{rows: rows, cols: cols, vals: make([]any, len(cols)), ptrs: make([]any, len(cols))}
|
||||
for i := range r.vals {
|
||||
r.ptrs[i] = &r.vals[i]
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *rowSource) Next() bool { return r.rows.Next() }
|
||||
|
||||
func (r *rowSource) Err() error { return r.rows.Err() }
|
||||
|
||||
func (r *rowSource) Values() ([]any, error) {
|
||||
if err := r.rows.Scan(r.ptrs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]any, len(r.vals))
|
||||
for i, v := range r.vals {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
converted, err := convert(v, r.cols[i].pgType)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("column %s: %w", r.cols[i].name, err)
|
||||
}
|
||||
out[i] = converted
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// convert adapts a non-NULL value to the target column type: MySQL and SQLite
|
||||
// return text and numbers as bytes and booleans as integers.
|
||||
func convert(v any, pgType string) (any, error) {
|
||||
switch pgType {
|
||||
case "boolean":
|
||||
return toBool(v)
|
||||
case "bytea":
|
||||
return toBytes(v), nil
|
||||
case "smallint", "integer", "bigint":
|
||||
return toInt64(v)
|
||||
case "real", "double precision":
|
||||
return toFloat64(v)
|
||||
case "numeric":
|
||||
return toNumeric(v)
|
||||
default:
|
||||
return toText(v), nil
|
||||
}
|
||||
}
|
||||
|
||||
func toBool(v any) (any, error) {
|
||||
switch b := v.(type) {
|
||||
case int64:
|
||||
return b != 0, nil
|
||||
case []byte:
|
||||
return strconv.ParseBool(string(b))
|
||||
case string:
|
||||
return strconv.ParseBool(b)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func toBytes(v any) any {
|
||||
if s, ok := v.(string); ok {
|
||||
return []byte(s)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func toInt64(v any) (any, error) {
|
||||
switch n := v.(type) {
|
||||
case uint64:
|
||||
if n > math.MaxInt64 {
|
||||
return nil, fmt.Errorf("value %d overflows bigint", n)
|
||||
}
|
||||
return int64(n), nil
|
||||
case []byte:
|
||||
return strconv.ParseInt(string(n), 10, 64)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func toFloat64(v any) (any, error) {
|
||||
if b, ok := v.([]byte); ok {
|
||||
return strconv.ParseFloat(string(b), 64)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func toNumeric(v any) (any, error) {
|
||||
var s string
|
||||
switch x := v.(type) {
|
||||
case []byte:
|
||||
s = string(x)
|
||||
case string:
|
||||
s = x
|
||||
default:
|
||||
return v, nil
|
||||
}
|
||||
var n pgtype.Numeric
|
||||
if err := n.Scan(s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func toText(v any) any {
|
||||
if b, ok := v.([]byte); ok {
|
||||
return string(b)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func queryStrings(ctx context.Context, db *sql.DB, query string, args ...any) ([]string, error) {
|
||||
rows, err := db.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var s string
|
||||
if err := rows.Scan(&s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func pgStrings(ctx context.Context, tx pgx.Tx, query string) ([]string, error) {
|
||||
rows, err := tx.Query(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pgx.CollectRows(rows, pgx.RowTo[string])
|
||||
}
|
||||
@@ -1,242 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/driver/postgres"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
|
||||
"github.com/netbirdio/netbird/idp/dex"
|
||||
"github.com/netbirdio/netbird/management/internals/shared/db"
|
||||
"github.com/netbirdio/netbird/management/server/activity"
|
||||
"github.com/netbirdio/netbird/management/server/store"
|
||||
)
|
||||
|
||||
type config struct {
|
||||
mysqlDSN string
|
||||
postgresDSN string
|
||||
eventsPostgresDSN string
|
||||
authPostgresDSN string
|
||||
eventsDB string
|
||||
authDB string
|
||||
mysqlTimezone string
|
||||
}
|
||||
|
||||
type storeMigration struct {
|
||||
name string
|
||||
source *sql.DB
|
||||
dialect dialect
|
||||
targetDSN string
|
||||
schema func(ctx context.Context, dsn string) error
|
||||
skipTables map[string]bool
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg, err := parseFlags(os.Args[1:])
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
// Silence the store constructors' migration logs.
|
||||
log.SetLevel(log.WarnLevel)
|
||||
|
||||
if err := run(context.Background(), cfg); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "migration failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func parseFlags(args []string) (*config, error) {
|
||||
cfg := &config{}
|
||||
fs := flag.NewFlagSet("netbird-mysql-migrate", flag.ContinueOnError)
|
||||
fs.StringVar(&cfg.mysqlDSN, "mysql-dsn", "", "MySQL DSN of the management store (required)")
|
||||
fs.StringVar(&cfg.postgresDSN, "postgres-dsn", "", "Postgres DSN to migrate all stores to (required)")
|
||||
fs.StringVar(&cfg.eventsPostgresDSN, "events-postgres-dsn", "", "Postgres DSN for activity events (default --postgres-dsn)")
|
||||
fs.StringVar(&cfg.authPostgresDSN, "auth-postgres-dsn", "", "Postgres DSN for the embedded IdP (default --postgres-dsn)")
|
||||
fs.StringVar(&cfg.eventsDB, "events-db", "/var/lib/netbird/events.db", "SQLite file of the activity events, skipped when missing")
|
||||
fs.StringVar(&cfg.authDB, "auth-db", "/var/lib/netbird/idp.db", "SQLite file of the embedded IdP, skipped when missing")
|
||||
fs.StringVar(&cfg.mysqlTimezone, "mysql-timezone", "UTC", "time zone the management server ran in, e.g. Europe/Berlin")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := time.LoadLocation(cfg.mysqlTimezone); err != nil {
|
||||
return nil, fmt.Errorf("--mysql-timezone: %w", err)
|
||||
}
|
||||
|
||||
if cfg.mysqlDSN == "" || cfg.postgresDSN == "" {
|
||||
return nil, errors.New("--mysql-dsn and --postgres-dsn are required")
|
||||
}
|
||||
if cfg.eventsPostgresDSN == "" {
|
||||
cfg.eventsPostgresDSN = cfg.postgresDSN
|
||||
}
|
||||
if cfg.authPostgresDSN == "" {
|
||||
cfg.authPostgresDSN = cfg.postgresDSN
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func run(ctx context.Context, cfg *config) error {
|
||||
migrations, closeSources, err := openSources(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeSources()
|
||||
|
||||
for _, m := range migrations {
|
||||
if err := m.schema(ctx, m.targetDSN); err != nil {
|
||||
return fmt.Errorf("create %s schema: %w", m.name, err)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
pools []*pgxpool.Pool
|
||||
txs []pgx.Tx
|
||||
)
|
||||
// Roll back first: pool Close waits for the connections open transactions hold.
|
||||
defer func() {
|
||||
for _, tx := range txs {
|
||||
_ = tx.Rollback(ctx)
|
||||
}
|
||||
for _, pool := range pools {
|
||||
pool.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
results := make([]string, 0, len(migrations))
|
||||
for _, m := range migrations {
|
||||
pool, err := pgxpool.New(ctx, m.targetDSN)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connect %s target: %w", m.name, err)
|
||||
}
|
||||
pools = append(pools, pool)
|
||||
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin %s transaction: %w", m.name, err)
|
||||
}
|
||||
txs = append(txs, tx)
|
||||
|
||||
tables, rows, err := copyStore(ctx, m.source, m.dialect, tx, m.skipTables)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", m.name, err)
|
||||
}
|
||||
results = append(results, fmt.Sprintf("%s: %d rows from %d tables", m.name, rows, tables))
|
||||
}
|
||||
|
||||
// Commit last so a failure in any store leaves every target empty.
|
||||
for i, tx := range txs {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("commit %s: %w", migrations[i].name, err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, r := range results {
|
||||
fmt.Fprintln(os.Stdout, r)
|
||||
}
|
||||
fmt.Fprintln(os.Stdout, "Done. Point the store, activity and auth store settings at Postgres before starting the server.")
|
||||
return nil
|
||||
}
|
||||
|
||||
func openSources(cfg *config) ([]storeMigration, func(), error) {
|
||||
var opened []*gorm.DB
|
||||
closeAll := func() {
|
||||
for _, g := range opened {
|
||||
if sqlDB, err := g.DB(); err == nil {
|
||||
_ = sqlDB.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
open := func(dialector gorm.Dialector) (*sql.DB, error) {
|
||||
g, err := gorm.Open(dialector, &gorm.Config{Logger: logger.Discard})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opened = append(opened, g)
|
||||
return g.DB()
|
||||
}
|
||||
|
||||
mysqlDB, err := open(mysql.Open(mysqlDSN(cfg.mysqlDSN, cfg.mysqlTimezone)))
|
||||
if err != nil {
|
||||
closeAll()
|
||||
return nil, nil, fmt.Errorf("open MySQL: %w", err)
|
||||
}
|
||||
migrations := []storeMigration{{
|
||||
name: "management", source: mysqlDB, dialect: mysqlDialect,
|
||||
targetDSN: cfg.postgresDSN, schema: createManagementSchema,
|
||||
}}
|
||||
|
||||
sqliteStores := []storeMigration{
|
||||
{name: "activity", targetDSN: cfg.eventsPostgresDSN, schema: createActivitySchema},
|
||||
{name: "auth", targetDSN: cfg.authPostgresDSN, schema: createAuthSchema, skipTables: map[string]bool{"migrations": true}},
|
||||
}
|
||||
for i, file := range []string{cfg.eventsDB, cfg.authDB} {
|
||||
m := sqliteStores[i]
|
||||
if _, err := os.Stat(file); errors.Is(err, os.ErrNotExist) {
|
||||
fmt.Fprintf(os.Stdout, "%s: %s not found, skipping\n", m.name, file)
|
||||
continue
|
||||
}
|
||||
m.source, err = open(sqlite.Open("file:" + file + "?mode=ro"))
|
||||
if err != nil {
|
||||
closeAll()
|
||||
return nil, nil, fmt.Errorf("open %s: %w", file, err)
|
||||
}
|
||||
m.dialect = sqliteDialect
|
||||
migrations = append(migrations, m)
|
||||
}
|
||||
|
||||
return migrations, closeAll, nil
|
||||
}
|
||||
|
||||
func mysqlDSN(dsn, timezone string) string {
|
||||
return strings.TrimSuffix(db.MysqlDSN(dsn), "&loc=Local") + "&loc=" + url.QueryEscape(timezone)
|
||||
}
|
||||
|
||||
func createManagementSchema(ctx context.Context, dsn string) error {
|
||||
s, err := store.NewPostgresqlStore(ctx, dsn, nil, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Close(ctx)
|
||||
}
|
||||
|
||||
func createActivitySchema(_ context.Context, dsn string) error {
|
||||
g, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: logger.Discard})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if sqlDB, err := g.DB(); err == nil {
|
||||
_ = sqlDB.Close()
|
||||
}
|
||||
}()
|
||||
return g.AutoMigrate(&activity.Event{}, &activity.DeletedUser{})
|
||||
}
|
||||
|
||||
func createAuthSchema(_ context.Context, dsn string) error {
|
||||
cfg := dex.Storage{Type: "postgres", Config: map[string]any{"dsn": dsn}}
|
||||
s, err := cfg.OpenStorage(slog.New(slog.DiscardHandler))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.Close()
|
||||
}
|
||||
@@ -1,225 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
dexstorage "github.com/dexidp/dex/storage"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/rs/xid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/netbirdio/netbird/idp/dex"
|
||||
"github.com/netbirdio/netbird/management/internals/shared/db"
|
||||
"github.com/netbirdio/netbird/management/server/activity"
|
||||
activitystore "github.com/netbirdio/netbird/management/server/activity/store"
|
||||
"github.com/netbirdio/netbird/management/server/store"
|
||||
"github.com/netbirdio/netbird/management/server/testutil"
|
||||
"github.com/netbirdio/netbird/management/server/types"
|
||||
"github.com/netbirdio/netbird/util/crypt"
|
||||
)
|
||||
|
||||
const testAccountID = "bf1c8084-ba50-4ce7-9439-34653001fc3b"
|
||||
|
||||
func TestRun(t *testing.T) {
|
||||
if runtime.GOOS == "windows" || (os.Getenv("CI") == "true" && runtime.GOOS != "linux") {
|
||||
t.Skip("needs Docker with Linux containers for MySQL and Postgres")
|
||||
}
|
||||
// Seed in UTC to match the --mysql-timezone default.
|
||||
time.Local = time.UTC
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
key, err := crypt.GenerateKey()
|
||||
require.NoError(t, err)
|
||||
|
||||
mysqlDSN := newMySQLDatabase(t, "run_src")
|
||||
pgDSN := newPostgresDatabase(t, ctx, "run_dst")
|
||||
mysqlStore := seedMySQL(t, ctx, mysqlDSN)
|
||||
eventID := seedEvents(t, ctx, dir, key)
|
||||
seedIdP(t, ctx, filepath.Join(dir, "idp.db"))
|
||||
|
||||
cfg, err := parseFlags([]string{
|
||||
"--mysql-dsn", mysqlDSN,
|
||||
"--postgres-dsn", pgDSN,
|
||||
"--events-db", filepath.Join(dir, "events.db"),
|
||||
"--auth-db", filepath.Join(dir, "idp.db"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, run(ctx, cfg))
|
||||
|
||||
t.Run("management", func(t *testing.T) {
|
||||
pgStore, err := store.NewPostgresqlStore(ctx, pgDSN, nil, true)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = pgStore.Close(ctx) })
|
||||
|
||||
assert.JSONEq(t, accountJSON(t, ctx, mysqlStore), accountJSON(t, ctx, pgStore),
|
||||
"the account read from Postgres must match the one in MySQL")
|
||||
})
|
||||
|
||||
t.Run("activity", func(t *testing.T) {
|
||||
t.Setenv("NB_ACTIVITY_EVENT_STORE_ENGINE", "postgres")
|
||||
t.Setenv("NB_ACTIVITY_EVENT_POSTGRES_DSN", pgDSN)
|
||||
events, err := activitystore.NewSqlStore(ctx, dir, key)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = events.Close(ctx) })
|
||||
|
||||
got, err := events.Get(ctx, testAccountID, 0, 10, false)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1, "the migrated event must be readable")
|
||||
assert.Equal(t, eventID, got[0].ID, "the event must keep its id")
|
||||
|
||||
next, err := events.Save(ctx, &activity.Event{Timestamp: time.Now(), AccountID: testAccountID, InitiatorID: "u", TargetID: "p"})
|
||||
require.NoError(t, err)
|
||||
assert.Greater(t, next.ID, eventID, "new events must continue after the migrated ids")
|
||||
})
|
||||
|
||||
t.Run("auth", func(t *testing.T) {
|
||||
idp := dex.Storage{Type: "postgres", Config: map[string]any{"dsn": pgDSN}}
|
||||
s, err := idp.OpenStorage(slog.New(slog.DiscardHandler))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
|
||||
pw, err := s.GetPassword(ctx, "admin@example.com")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []byte("hash"), pw.Hash, "the IdP password must survive the migration")
|
||||
})
|
||||
|
||||
t.Run("refuses a target that holds data", func(t *testing.T) {
|
||||
assert.ErrorContains(t, run(ctx, cfg), "target already holds data")
|
||||
})
|
||||
}
|
||||
|
||||
func seedMySQL(t *testing.T, ctx context.Context, dsn string) *store.SqlStore {
|
||||
t.Helper()
|
||||
t.Setenv("NETBIRD_STORE_ENGINE", string(types.SqliteStoreEngine))
|
||||
seed, cleanup, err := store.NewTestStoreFromSQL(ctx, "../../management/server/testdata/extended-store.sql", t.TempDir())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(cleanup)
|
||||
|
||||
mysqlStore, err := store.NewMysqlStoreFromSqlStore(ctx, seed.(*store.SqlStore), dsn, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = mysqlStore.Close(ctx) })
|
||||
return mysqlStore
|
||||
}
|
||||
|
||||
func seedEvents(t *testing.T, ctx context.Context, dir, key string) uint64 {
|
||||
t.Helper()
|
||||
events, err := activitystore.NewSqlStore(ctx, dir, key)
|
||||
require.NoError(t, err)
|
||||
defer events.Close(ctx)
|
||||
|
||||
event, err := events.Save(ctx, &activity.Event{
|
||||
Timestamp: time.Now(), Activity: activity.PeerAddedByUser,
|
||||
AccountID: testAccountID, InitiatorID: "user", TargetID: "peer",
|
||||
Meta: map[string]any{"name": "peer"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return event.ID
|
||||
}
|
||||
|
||||
func seedIdP(t *testing.T, ctx context.Context, file string) {
|
||||
t.Helper()
|
||||
idp := dex.Storage{Type: "sqlite3", Config: map[string]any{"file": file}}
|
||||
s, err := idp.OpenStorage(slog.New(slog.DiscardHandler))
|
||||
require.NoError(t, err)
|
||||
defer s.Close()
|
||||
|
||||
require.NoError(t, s.CreatePassword(ctx, dexstorage.Password{
|
||||
Email: "admin@example.com", Hash: []byte("hash"), Username: "admin", UserID: "admin-id",
|
||||
}))
|
||||
}
|
||||
|
||||
func accountJSON(t *testing.T, ctx context.Context, s store.Store) string {
|
||||
t.Helper()
|
||||
var account *types.Account
|
||||
require.NoError(t, s.ExecuteInTransaction(ctx, func(tx store.Store) error {
|
||||
var err error
|
||||
account, err = tx.GetAccount(ctx, testAccountID)
|
||||
return err
|
||||
}))
|
||||
out, err := json.Marshal(account)
|
||||
require.NoError(t, err)
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func TestMySQLTimezone(t *testing.T) {
|
||||
if runtime.GOOS == "windows" || (os.Getenv("CI") == "true" && runtime.GOOS != "linux") {
|
||||
t.Skip("needs Docker with Linux containers for MySQL and Postgres")
|
||||
}
|
||||
ctx := context.Background()
|
||||
mysqlDSN := newMySQLDatabase(t, "tz_src")
|
||||
pgDSN := newPostgresDatabase(t, ctx, "tz_dst")
|
||||
ms := seedMySQL(t, ctx, mysqlDSN)
|
||||
require.NoError(t, ms.GetDB().Exec("UPDATE accounts SET created_at = '2026-01-02 03:04:05' WHERE id = ?", testAccountID).Error)
|
||||
|
||||
cfg, err := parseFlags([]string{
|
||||
"--mysql-dsn", mysqlDSN,
|
||||
"--postgres-dsn", pgDSN,
|
||||
"--mysql-timezone", "Asia/Tokyo",
|
||||
"--events-db", filepath.Join(t.TempDir(), "none.db"),
|
||||
"--auth-db", filepath.Join(t.TempDir(), "none.db"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, run(ctx, cfg))
|
||||
|
||||
pool, err := pgxpool.New(ctx, pgDSN)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(pool.Close)
|
||||
var got time.Time
|
||||
require.NoError(t, pool.QueryRow(ctx, "SELECT created_at FROM accounts WHERE id = $1", testAccountID).Scan(&got))
|
||||
want := time.Date(2026, 1, 1, 18, 4, 5, 0, time.UTC)
|
||||
assert.True(t, want.Equal(got), "created_at must keep the instant the server wrote, got %s", got.UTC())
|
||||
}
|
||||
|
||||
func TestParseFlagsRejectsUnknownTimezone(t *testing.T) {
|
||||
_, err := parseFlags([]string{"--mysql-dsn", "m", "--postgres-dsn", "p", "--mysql-timezone", "Mars/Base"})
|
||||
assert.ErrorContains(t, err, "--mysql-timezone")
|
||||
}
|
||||
|
||||
// newMySQLDatabase creates a database unique to this run in the shared
|
||||
// container, so repeated runs (-count) don't collide, and drops it afterwards.
|
||||
func newMySQLDatabase(t *testing.T, prefix string) string {
|
||||
t.Helper()
|
||||
_, base, err := testutil.CreateMysqlTestContainer()
|
||||
require.NoError(t, err)
|
||||
admin, err := gorm.Open(mysql.Open(db.MysqlDSN(base)), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
name := prefix + "_" + xid.New().String()
|
||||
require.NoError(t, admin.Exec("CREATE DATABASE "+name).Error)
|
||||
t.Cleanup(func() {
|
||||
_ = admin.Exec("DROP DATABASE IF EXISTS " + name).Error
|
||||
if sqlDB, err := admin.DB(); err == nil {
|
||||
_ = sqlDB.Close()
|
||||
}
|
||||
})
|
||||
return base[:strings.LastIndex(base, "/")] + "/" + name
|
||||
}
|
||||
|
||||
// newPostgresDatabase is the Postgres counterpart of newMySQLDatabase.
|
||||
func newPostgresDatabase(t *testing.T, ctx context.Context, prefix string) string {
|
||||
t.Helper()
|
||||
_, base, err := testutil.CreatePostgresTestContainer()
|
||||
require.NoError(t, err)
|
||||
// Dex connects through lib/pq, which requires TLS unless told otherwise.
|
||||
base = strings.TrimSuffix(base, "?")
|
||||
admin, err := pgxpool.New(ctx, base+"?sslmode=disable")
|
||||
require.NoError(t, err)
|
||||
name := prefix + "_" + xid.New().String()
|
||||
_, err = admin.Exec(ctx, "CREATE DATABASE "+name)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
_, _ = admin.Exec(context.Background(), "DROP DATABASE IF EXISTS "+name+" WITH (FORCE)")
|
||||
admin.Close()
|
||||
})
|
||||
return base[:strings.LastIndex(base, "/")] + "/" + name + "?sslmode=disable"
|
||||
}
|
||||
Reference in New Issue
Block a user