diff --git a/.github/workflows/check-license-dependencies.yml b/.github/workflows/check-license-dependencies.yml index 27c59f8d6..81d293e4f 100644 --- a/.github/workflows/check-license-dependencies.yml +++ b/.github/workflows/check-license-dependencies.yml @@ -34,7 +34,7 @@ jobs: while IFS= read -r dir; do echo "=== Checking $dir ===" # Search for problematic imports, excluding test files - RESULTS=$(grep -r "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\)" "$dir" --include="*.go" 2>/dev/null | grep -v "_test.go" | grep -v "test_" | grep -v "/test/" | grep -v "tools/idp-migrate/" | grep -v "tools/mysql-migrate/" || true) + RESULTS=$(grep -r "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\)" "$dir" --include="*.go" 2>/dev/null | grep -v "_test.go" | grep -v "test_" | grep -v "/test/" | grep -v "tools/idp-migrate/" || true) if [ -n "$RESULTS" ]; then echo "❌ Found problematic dependencies:" echo "$RESULTS" @@ -93,7 +93,7 @@ jobs: IMPORTERS=$(go list -json -deps ./... 2>/dev/null | jq -r "select(.Imports[]? == \"$package\") | .ImportPath") # Check if any importer is NOT in management/signal/relay - BSD_IMPORTER=$(echo "$IMPORTERS" | grep -v "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\|combined\|tools/idp-migrate\|tools/mysql-migrate\)" | head -1) + BSD_IMPORTER=$(echo "$IMPORTERS" | grep -v "github.com/netbirdio/netbird/\(management\|signal\|relay\|proxy\|combined\|tools/idp-migrate\)" | head -1) if [ -n "$BSD_IMPORTER" ]; then echo "❌ $package ($license) is imported by BSD-licensed code: $BSD_IMPORTER" diff --git a/.github/workflows/golang-test-linux.yml b/.github/workflows/golang-test-linux.yml index 880370ede..b3e4dd4f7 100644 --- a/.github/workflows/golang-test-linux.yml +++ b/.github/workflows/golang-test-linux.yml @@ -307,7 +307,7 @@ jobs: sh -c ' \ apk update; apk add --no-cache \ ca-certificates iptables ip6tables dbus dbus-dev libpcap-dev build-base; \ - go test -buildvcs=false -tags "devcert privileged" -v -timeout 10m -p 1 $(go list -e -buildvcs=false ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/ui -e /upload-server -e /client/testutil/privileged -e tools/mysql-migrate) + go test -buildvcs=false -tags "devcert privileged" -v -timeout 10m -p 1 $(go list -e -buildvcs=false ./... | grep -v -e /management -e /signal -e /relay -e /proxy -e /combined -e /client/ui -e /upload-server -e /client/testutil/privileged) ' test_relay: @@ -498,7 +498,7 @@ jobs: fail-fast: false matrix: arch: ["amd64"] - store: ["sqlite", "postgres", "mysql"] + store: ["sqlite", "postgres"] runs-on: ubuntu-22.04 steps: - name: Checkout code @@ -547,10 +547,6 @@ jobs: DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} run: echo "$DOCKER_TOKEN" | sudo docker login --username "$DOCKER_USER" --password-stdin - - name: download mysql image - if: matrix.store == 'mysql' - run: docker pull mlsmaycon/warmed-mysql:8 - # The -json stream goes through tools/gotestsummary so the log shows one # line per test, the output of failed tests, the head of a timeout panic # with the still-running tests, and the slowest tests per package. diff --git a/.github/workflows/golang-test-windows.yml b/.github/workflows/golang-test-windows.yml index 49be3af58..ca300f7df 100644 --- a/.github/workflows/golang-test-windows.yml +++ b/.github/workflows/golang-test-windows.yml @@ -76,7 +76,7 @@ jobs: # resolve; the Where-Object pipeline then drops the broken package by # path. Without -e, go list aborts with empty stdout. run: | - $packages = go list -e ./... | Where-Object { $_ -notmatch '/management' } | Where-Object { $_ -notmatch '/relay' } | Where-Object { $_ -notmatch '/signal' } | Where-Object { $_ -notmatch '/proxy' } | Where-Object { $_ -notmatch '/combined' } | Where-Object { $_ -notmatch '/client/ui$' } | Where-Object { $_ -notmatch '/tools/mysql-migrate' } + $packages = go list -e ./... | Where-Object { $_ -notmatch '/management' } | Where-Object { $_ -notmatch '/relay' } | Where-Object { $_ -notmatch '/signal' } | Where-Object { $_ -notmatch '/proxy' } | Where-Object { $_ -notmatch '/combined' } | Where-Object { $_ -notmatch '/client/ui$' } $goExe = "C:\hostedtoolcache\windows\go\${{ steps.go.outputs.go-version }}\x64\bin\go.exe" $cmd = "$goExe test -tags `"devcert privileged`" -timeout 10m -p 1 $($packages -join ' ') > test-out.txt 2>&1" Set-Content -Path "${{ github.workspace }}\run-tests.cmd" -Value $cmd diff --git a/.github/workflows/test-infrastructure-files.yml b/.github/workflows/test-infrastructure-files.yml index 4875adc31..68cf1b639 100644 --- a/.github/workflows/test-infrastructure-files.yml +++ b/.github/workflows/test-infrastructure-files.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - store: ["sqlite", "postgres", "mysql"] + store: ["sqlite", "postgres"] services: postgres: image: ${{ (matrix.store == 'postgres') && 'postgres' || '' }} @@ -35,19 +35,6 @@ jobs: --health-timeout 5s ports: - 5432:5432 - mysql: - image: ${{ (matrix.store == 'mysql') && 'mysql' || '' }} - env: - MYSQL_USER: netbird - MYSQL_PASSWORD: mysql - MYSQL_ROOT_PASSWORD: mysqlroot - MYSQL_DATABASE: netbird - options: >- - --health-cmd "mysqladmin ping --silent" - --health-interval 10s - --health-timeout 5s - ports: - - 3306:3306 steps: - name: Set Database Connection String run: | @@ -56,11 +43,6 @@ jobs: else echo "NETBIRD_STORE_ENGINE_POSTGRES_DSN==" >> $GITHUB_ENV fi - if [ "${{ matrix.store }}" == "mysql" ]; then - echo "NETBIRD_STORE_ENGINE_MYSQL_DSN=netbird:mysql@tcp($(hostname -I | awk '{print $1}'):3306)/netbird" >> $GITHUB_ENV - else - echo "NETBIRD_STORE_ENGINE_MYSQL_DSN==" >> $GITHUB_ENV - fi - name: Install jq run: sudo apt-get install -y jq @@ -86,15 +68,6 @@ jobs: restore-keys: | ${{ runner.os }}-go- - - name: Setup MySQL privileges - if: matrix.store == 'mysql' - run: | - sleep 10 - mysql -h 127.0.0.1 -u root -pmysqlroot -e " - GRANT SYSTEM_VARIABLES_ADMIN ON *.* TO 'netbird'@'%'; - FLUSH PRIVILEGES; - " - - name: cp setup.env run: cp infrastructure_files/tests/setup.env infrastructure_files/ @@ -114,7 +87,6 @@ jobs: CI_NETBIRD_AUTH_SUPPORTED_SCOPES: "openid profile email offline_access api email_verified" CI_NETBIRD_STORE_CONFIG_ENGINE: ${{ matrix.store }} NETBIRD_STORE_ENGINE_POSTGRES_DSN: ${{ env.NETBIRD_STORE_ENGINE_POSTGRES_DSN }} - NETBIRD_STORE_ENGINE_MYSQL_DSN: ${{ env.NETBIRD_STORE_ENGINE_MYSQL_DSN }} CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false - name: check values @@ -143,7 +115,6 @@ jobs: CI_NETBIRD_SIGNAL_PORT: 12345 CI_NETBIRD_STORE_CONFIG_ENGINE: ${{ matrix.store }} NETBIRD_STORE_ENGINE_POSTGRES_DSN: "${{ env.NETBIRD_STORE_ENGINE_POSTGRES_DSN }}$" - NETBIRD_STORE_ENGINE_MYSQL_DSN: "${{ env.NETBIRD_STORE_ENGINE_MYSQL_DSN }}$" CI_NETBIRD_MGMT_IDP_SIGNKEY_REFRESH: false CI_NETBIRD_TURN_EXTERNAL_IP: "1.2.3.4" CI_NETBIRD_MGMT_DISABLE_DEFAULT_POLICY: false @@ -182,7 +153,6 @@ jobs: grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep Scope | grep "$CI_NETBIRD_AUTH_SUPPORTED_SCOPES" grep -A 10 PKCEAuthorizationFlow management.json | grep -A 10 ProviderConfig | grep -A 3 RedirectURLs | grep "http://localhost:53000" grep "external-ip" turnserver.conf | grep $CI_NETBIRD_TURN_EXTERNAL_IP - grep "NETBIRD_STORE_ENGINE_MYSQL_DSN=$NETBIRD_STORE_ENGINE_MYSQL_DSN" docker-compose.yml grep NETBIRD_STORE_ENGINE_POSTGRES_DSN docker-compose.yml | egrep "$NETBIRD_STORE_ENGINE_POSTGRES_DSN" # check relay values grep "NB_EXPOSED_ADDRESS=rels://$CI_NETBIRD_DOMAIN:33445" docker-compose.yml diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 41c330a64..62b64717e 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -280,28 +280,6 @@ builds: - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser mod_timestamp: "{{ .CommitTimestamp }}" - - id: netbird-mysql-migrate - dir: tools/mysql-migrate - env: - - CGO_ENABLED=1 - - >- - {{- if eq .Runtime.Goos "linux" }} - {{- if eq .Arch "arm64"}}CC=aarch64-linux-gnu-gcc{{- end }} - {{- if eq .Arch "arm"}}CC=arm-linux-gnueabihf-gcc{{- end }} - {{- end }} - binary: netbird-mysql-migrate - goos: - - linux - goarch: - - amd64 - - arm64 - - arm - goarm: - - 7 - ldflags: - - -s -w -X github.com/netbirdio/netbird/version.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser - mod_timestamp: "{{ .CommitTimestamp }}" - universal_binaries: - id: netbird @@ -318,10 +296,6 @@ archives: builds: - netbird-idp-migrate name_template: "netbird-idp-migrate_{{ .Version }}_{{ .Os }}_{{ .Arch }}" - - id: netbird-mysql-migrate - builds: - - netbird-mysql-migrate - name_template: "netbird-mysql-migrate_{{ .Version }}_{{ .Os }}_{{ .Arch }}" - id: nblink builds: - nblink diff --git a/combined/cmd/admin_config_test.go b/combined/cmd/admin_config_test.go index ff7045d38..5cec26552 100644 --- a/combined/cmd/admin_config_test.go +++ b/combined/cmd/admin_config_test.go @@ -35,13 +35,9 @@ func TestOpenAdminEventStoreMissingEncryptionKeyReturnsNilInterface(t *testing.T func TestApplyServerStoreEnv(t *testing.T) { t.Setenv("NB_STORE_ENGINE_POSTGRES_DSN", "") - t.Setenv("NB_STORE_ENGINE_MYSQL_DSN", "") t.Setenv("NB_STORE_ENGINE_SQLITE_FILE", "") applyServerStoreEnv(StoreConfig{Engine: "postgres", DSN: "postgres-dsn", File: "store.db"}) require.Equal(t, "postgres-dsn", os.Getenv("NB_STORE_ENGINE_POSTGRES_DSN")) require.Equal(t, "store.db", os.Getenv("NB_STORE_ENGINE_SQLITE_FILE")) - - applyServerStoreEnv(StoreConfig{Engine: "mysql", DSN: "mysql-dsn"}) - require.Equal(t, "mysql-dsn", os.Getenv("NB_STORE_ENGINE_MYSQL_DSN")) } diff --git a/combined/cmd/config.go b/combined/cmd/config.go index 890c86876..ca3b25617 100644 --- a/combined/cmd/config.go +++ b/combined/cmd/config.go @@ -188,7 +188,7 @@ type RelaysConfig struct { type StoreConfig struct { Engine string `yaml:"engine"` EncryptionKey string `yaml:"encryptionKey"` - DSN string `yaml:"dsn"` // Connection string for postgres or mysql engines + DSN string `yaml:"dsn"` // Connection string for the postgres engine File string `yaml:"file"` // SQLite database file path (optional, defaults to dataDir) } diff --git a/combined/cmd/root.go b/combined/cmd/root.go index fa65f7b6d..fedb0e1cc 100644 --- a/combined/cmd/root.go +++ b/combined/cmd/root.go @@ -127,13 +127,8 @@ func execute(cmd *cobra.Command, _ []string) error { // initializeConfig loads and validates the configuration, then initializes logging. func applyServerStoreEnv(storeConfig StoreConfig) { - if dsn := storeConfig.DSN; dsn != "" { - switch strings.ToLower(storeConfig.Engine) { - case "postgres": - os.Setenv("NB_STORE_ENGINE_POSTGRES_DSN", dsn) - case "mysql": - os.Setenv("NB_STORE_ENGINE_MYSQL_DSN", dsn) - } + if dsn := storeConfig.DSN; dsn != "" && strings.EqualFold(storeConfig.Engine, "postgres") { + os.Setenv("NB_STORE_ENGINE_POSTGRES_DSN", dsn) } if file := storeConfig.File; file != "" { os.Setenv("NB_STORE_ENGINE_SQLITE_FILE", file) diff --git a/combined/config.yaml.example b/combined/config.yaml.example index 085e4344f..68dbf701e 100644 --- a/combined/config.yaml.example +++ b/combined/config.yaml.example @@ -110,8 +110,8 @@ server: # Store configuration store: - engine: "sqlite" # sqlite, postgres, or mysql - dsn: "" # Connection string for postgres or mysql + engine: "sqlite" # sqlite or postgres + dsn: "" # Connection string for postgres encryptionKey: "" # file: "" # Custom SQLite file path (optional, defaults to {dataDir}/store.db) diff --git a/docs/agent-networks/modules/20-management-store.md b/docs/agent-networks/modules/20-management-store.md index 1acc12611..658b1fb4f 100644 --- a/docs/agent-networks/modules/20-management-store.md +++ b/docs/agent-networks/modules/20-management-store.md @@ -64,17 +64,17 @@ Reads decrypt provider secrets in-place; writes do `provider.Copy().EncryptSensi ### Correctness - `SaveAgentNetworkProvider` saves the copy (sql_store_agentnetwork.go:95). The caller's in-memory pointer therefore keeps plaintext `api_key` and any `CreatedAt`/`UpdatedAt` gorm autofills land on the copy, not the original. Callers that need synced timestamps must re-fetch. -- `IncrementAgentNetworkConsumption`'s `Create` provides initial counter values (`TokensInput: tokensIn`, etc.) in the row, and on conflict the assignments add the same deltas to the existing values. The insert-vs-update arithmetic is consistent. Cross-check that no engine in use (sqlite, postgres, mysql) silently rejects the `OnConflict` clause — GORM emits engine-specific SQL but `ON DUPLICATE KEY UPDATE` (mysql) vs `ON CONFLICT (...)` (sqlite/postgres) need their unique constraint to match the composite PK on `agent_network_consumption`; it does, by construction. +- `IncrementAgentNetworkConsumption`'s `Create` provides initial counter values (`TokensInput: tokensIn`, etc.) in the row, and on conflict the assignments add the same deltas to the existing values. The insert-vs-update arithmetic is consistent. Cross-check that no engine in use (sqlite, postgres) silently rejects the `OnConflict` clause — GORM emits `ON CONFLICT (...)`, which needs its unique constraint to match the composite PK on `agent_network_consumption`; it does, by construction. - `IncrementAgentNetworkConsumption` writes `updated_at: time.Now().UTC()` literally inside the assignments map (sql_store_agentnetwork.go:333) — fine, but it's a Go-side timestamp captured at call time, not a DB-side `now()`. Acceptable for an audit field. - `GetAgentNetworkConsumption` returns a zero-valued non-nil row on `ErrRecordNotFound` (sql_store_agentnetwork.go:364-371). Document or rename — a typed sentinel error would be more orthodox; callers must know not to error-check. ### Concurrency / transactions -- Hot-path `IncrementAgentNetworkConsumption` runs outside any explicit transaction; concurrency safety relies entirely on the DB serialising the `ON CONFLICT` upsert against the composite PK. This is correct for postgres and mysql; for sqlite it serialises behind the single writer. +- Hot-path `IncrementAgentNetworkConsumption` runs outside any explicit transaction; concurrency safety relies entirely on the DB serialising the `ON CONFLICT` upsert against the composite PK. This is correct for postgres; for sqlite it serialises behind the single writer. - `SaveAgentNetworkSettings` is a blind upsert with no version/etag — concurrent writes from two operators last-write-wins on the collection-toggle flags (settings.go:23-25). Acceptable for admin-curated state but worth flagging. - `Save*Provider` uses `db.Save` on a struct with a PK already set — GORM emits UPDATE or INSERT based on row existence. No upsert clause is attached, so a race between two creates with the same generated `xid` (vanishingly unlikely) would surface as a PK violation. ### Migration safety -- All six tables ride `AutoMigrate` (sql_store.go:141-142). AutoMigrate is additive: new columns get added, but it never drops columns nor narrows types. Three `bool` columns on `agent_network_settings` (`EnableLogCollection`, `EnablePromptCollection`, `RedactPii`) default to false at the GORM/DDL layer for existing rows; the test at sql_store_agentnetwork_budgetrule_test.go:83-112 locks that down on a fresh sqlite. Verify postgres/mysql produce the same default. +- All six tables ride `AutoMigrate` (sql_store.go:141-142). AutoMigrate is additive: new columns get added, but it never drops columns nor narrows types. Three `bool` columns on `agent_network_settings` (`EnableLogCollection`, `EnablePromptCollection`, `RedactPii`) default to false at the GORM/DDL layer for existing rows; the test at sql_store_agentnetwork_budgetrule_test.go:83-112 locks that down on a fresh sqlite. Verify postgres produces the same default. - The named index `idx_agent_network_settings_cluster_subdomain` on settings.go:15 is declared on only `subdomain`. Either the cluster column also needs `gorm:"index:idx_agent_network_settings_cluster_subdomain"` to make it composite, or the name is misleading. - The named index `idx_agent_network_provider` on `Provider.ProviderID` (provider.go:30) is *not* unique and not scoped to account — two providers in the same account with the same `provider_id` are permitted at the DB layer; uniqueness, if any, must live above the store. @@ -101,7 +101,7 @@ Gap: there is no store-level test for providers (encryption round-trip), policie - No retention / GC for `agent_network_consumption`. - No `Delete` for `Settings` (one row per account, cleared with the account). -- No DB-engine-specific tuning — the same struct tags drive sqlite, mysql, postgres. +- No DB-engine-specific tuning — the same struct tags drive sqlite and postgres. - Provider `extra_values` and `models` are JSON blobs; querying inside them is not supported by design. - `GetAgentNetworkConsumption` "not-found = zero row" contract is convenient but unconventional. diff --git a/go.mod b/go.mod index bc8e5e287..beb612a72 100644 --- a/go.mod +++ b/go.mod @@ -113,7 +113,6 @@ require ( github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 github.com/stretchr/testify v1.12.1 github.com/testcontainers/testcontainers-go v0.37.0 - github.com/testcontainers/testcontainers-go/modules/mysql v0.37.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.37.0 github.com/testcontainers/testcontainers-go/modules/redis v0.37.0 github.com/things-go/go-socks5 v0.0.4 @@ -142,7 +141,6 @@ require ( google.golang.org/api v0.276.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 gopkg.in/yaml.v3 v3.0.1 - gorm.io/driver/mysql v1.5.7 gorm.io/driver/postgres v1.5.7 gorm.io/driver/sqlite v1.5.7 gorm.io/gorm v1.25.12 diff --git a/go.sum b/go.sum index 9af59daf5..82e2daf78 100644 --- a/go.sum +++ b/go.sum @@ -252,7 +252,6 @@ github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3yg github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6 h1:teYtXy9B7y5lHTp8V9KPxpYRAVA7dozigQcMiBust1s= github.com/go-quicktest/qt v1.101.1-0.20240301121107-c6c8733fa1e6/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= -github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= @@ -721,8 +720,6 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/testcontainers/testcontainers-go v0.37.0 h1:L2Qc0vkTw2EHWQ08djon0D2uw7Z/PtHS/QzZZ5Ra/hg= github.com/testcontainers/testcontainers-go v0.37.0/go.mod h1:QPzbxZhQ6Bclip9igjLFj6z0hs01bU8lrl2dHQmgFGM= -github.com/testcontainers/testcontainers-go/modules/mysql v0.37.0 h1:LqUos1oR5iuuzorFnSvxsHNdYdCHB/DfI82CuT58wbI= -github.com/testcontainers/testcontainers-go/modules/mysql v0.37.0/go.mod h1:vHEEHx5Kf+uq5hveaVAMrTzPY8eeRZcKcl23MRw5Tkc= github.com/testcontainers/testcontainers-go/modules/postgres v0.37.0 h1:hsVwFkS6s+79MbKEO+W7A1wNIw1fmkMtF4fg83m6kbc= github.com/testcontainers/testcontainers-go/modules/postgres v0.37.0/go.mod h1:Qj/eGbRbO/rEYdcRLmN+bEojzatP/+NS1y8ojl2PQsc= github.com/testcontainers/testcontainers-go/modules/redis v0.37.0 h1:9HIY28I9ME/Zmb+zey1p/I1mto5+5ch0wLX+nJdOsQ4= @@ -1025,13 +1022,10 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= -gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM= gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA= gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I= gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= -gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= diff --git a/infrastructure_files/configure.sh b/infrastructure_files/configure.sh index ce1a041e6..2d9fe5ee8 100755 --- a/infrastructure_files/configure.sh +++ b/infrastructure_files/configure.sh @@ -53,18 +53,6 @@ if [[ "$NETBIRD_STORE_CONFIG_ENGINE" == "postgres" ]]; then export NETBIRD_STORE_ENGINE_POSTGRES_DSN fi -# Check if MySQL is set as the store engine -if [[ "$NETBIRD_STORE_CONFIG_ENGINE" == "mysql" ]]; then - # Exit if 'NETBIRD_STORE_ENGINE_MYSQL_DSN' is not set - if [[ -z "$NETBIRD_STORE_ENGINE_MYSQL_DSN" ]]; then - echo "Warning: NETBIRD_STORE_CONFIG_ENGINE=mysql but NETBIRD_STORE_ENGINE_MYSQL_DSN is not set." - echo "Please add the following line to your setup.env file:" - echo 'NETBIRD_STORE_ENGINE_MYSQL_DSN=":@tcp(127.0.0.1:3306)/"' - exit 1 - fi - export NETBIRD_STORE_ENGINE_MYSQL_DSN -fi - # local development or tests if [[ $NETBIRD_DOMAIN == "localhost" || $NETBIRD_DOMAIN == "127.0.0.1" ]]; then export NETBIRD_MGMT_SINGLE_ACCOUNT_MODE_DOMAIN="netbird.selfhosted" diff --git a/infrastructure_files/docker-compose.yml.tmpl b/infrastructure_files/docker-compose.yml.tmpl index 1c9c63f78..0ffb20a72 100644 --- a/infrastructure_files/docker-compose.yml.tmpl +++ b/infrastructure_files/docker-compose.yml.tmpl @@ -94,7 +94,6 @@ services: ] environment: - NETBIRD_STORE_ENGINE_POSTGRES_DSN=$NETBIRD_STORE_ENGINE_POSTGRES_DSN - - NETBIRD_STORE_ENGINE_MYSQL_DSN=$NETBIRD_STORE_ENGINE_MYSQL_DSN # Coturn coturn: diff --git a/infrastructure_files/docker-compose.yml.tmpl.traefik b/infrastructure_files/docker-compose.yml.tmpl.traefik index 0010974c5..7fcf1cdde 100644 --- a/infrastructure_files/docker-compose.yml.tmpl.traefik +++ b/infrastructure_files/docker-compose.yml.tmpl.traefik @@ -100,7 +100,6 @@ services: - traefik.http.services.netbird-management.loadbalancer.server.scheme=h2c environment: - NETBIRD_STORE_ENGINE_POSTGRES_DSN=$NETBIRD_STORE_ENGINE_POSTGRES_DSN - - NETBIRD_STORE_ENGINE_MYSQL_DSN=$NETBIRD_STORE_ENGINE_MYSQL_DSN # Coturn coturn: diff --git a/infrastructure_files/migrate-to-enterprise.sh b/infrastructure_files/migrate-to-enterprise.sh index 8abde2f68..5b2cc2f79 100755 --- a/infrastructure_files/migrate-to-enterprise.sh +++ b/infrastructure_files/migrate-to-enterprise.sh @@ -855,7 +855,7 @@ configure_sqlite_store() { POSTGRES_DSN="host=postgres user=netbird password=${POSTGRES_PASSWORD} dbname=netbird port=5432 sslmode=disable" } -# mysql, or something this script has never seen. Swapping the images is still +# An engine this script has never seen. Swapping the images is still # valid; touching the store is not. configure_unsupported_store() { MIGRATE_POSTGRES="no" diff --git a/infrastructure_files/migrate.sh b/infrastructure_files/migrate.sh index 46971a57a..2b018ab54 100755 --- a/infrastructure_files/migrate.sh +++ b/infrastructure_files/migrate.sh @@ -472,12 +472,6 @@ detect_store_config() { if [[ -n "$pg_dsn" ]]; then STORE_DSN="$pg_dsn" fi - - local mysql_dsn - mysql_dsn=$(grep '^NETBIRD_STORE_ENGINE_MYSQL_DSN=' "$INSTALL_DIR/setup.env" 2>/dev/null | sed 's/^NETBIRD_STORE_ENGINE_MYSQL_DSN=//' | tr -d '"' || echo "") - if [[ -n "$mysql_dsn" ]]; then - STORE_DSN="$mysql_dsn" - fi fi # Also check base.setup.env @@ -487,12 +481,6 @@ detect_store_config() { if [[ -n "$pg_dsn" ]]; then STORE_DSN="$pg_dsn" fi - - local mysql_dsn - mysql_dsn=$(grep '^NETBIRD_STORE_ENGINE_MYSQL_DSN=' "$INSTALL_DIR/base.setup.env" 2>/dev/null | sed 's/^NETBIRD_STORE_ENGINE_MYSQL_DSN=//' | tr -d '"' || echo "") - if [[ -n "$mysql_dsn" ]]; then - STORE_DSN="$mysql_dsn" - fi fi log_success "Store engine: $STORE_ENGINE" diff --git a/management/internals/modules/agentnetwork/agent_config_realstore_test.go b/management/internals/modules/agentnetwork/agent_config_realstore_test.go index 2fd541fad..e69cec59a 100644 --- a/management/internals/modules/agentnetwork/agent_config_realstore_test.go +++ b/management/internals/modules/agentnetwork/agent_config_realstore_test.go @@ -341,7 +341,7 @@ func TestGetAgentConfigForUser_RealStore(t *testing.T) { require.NoError(t, s.SaveAgentNetworkPolicy(ctx, newSynthTestPolicy(provider.ID, "grp-eng", ""))) // users.account_id is a foreign key into accounts, enforced on - // MySQL/Postgres, so the account row must exist before its users. + // Postgres, so the account row must exist before its users. require.NoError(t, s.SaveAccount(ctx, &nbtypes.Account{Id: testAccountID})) require.NoError(t, s.SaveUser(ctx, &nbtypes.User{ Id: "user-in", AccountID: testAccountID, Role: nbtypes.UserRoleUser, AutoGroups: []string{"grp-eng"}, diff --git a/management/internals/modules/agentnetwork/manager.go b/management/internals/modules/agentnetwork/manager.go index 71ff53214..69ae4b432 100644 --- a/management/internals/modules/agentnetwork/manager.go +++ b/management/internals/modules/agentnetwork/manager.go @@ -1277,7 +1277,6 @@ func isUniqueConstraintError(err error) bool { } msg := err.Error() return strings.Contains(msg, "(SQLSTATE 23505)") || // postgres - strings.Contains(msg, "Error 1062 (23000)") || // mysql strings.Contains(msg, "UNIQUE constraint failed") // sqlite } diff --git a/management/internals/modules/agentnetwork/provider_redaction_test.go b/management/internals/modules/agentnetwork/provider_redaction_test.go index d6a749fb9..1503f4121 100644 --- a/management/internals/modules/agentnetwork/provider_redaction_test.go +++ b/management/internals/modules/agentnetwork/provider_redaction_test.go @@ -98,9 +98,8 @@ func TestGetProvider_RedactsForReadOnlyViewer(t *testing.T) { // keeps the account-wide view with full config. // newSelfScopeStore seeds the account and its users only, so each test -// declares exactly the providers and policies it asserts on — the store -// rejects re-saving a policy id on MySQL, so tests never overwrite each -// other's rows. +// declares exactly the providers and policies it asserts on, so tests never +// overwrite each other's rows. func newSelfScopeStore(t *testing.T) (*managerImpl, store.Store) { t.Helper() mgr, s := newAgentConfigTestMgr(t) diff --git a/management/internals/modules/agentnetwork/types/settings.go b/management/internals/modules/agentnetwork/types/settings.go index 70ecc5f70..6a28e6a5c 100644 --- a/management/internals/modules/agentnetwork/types/settings.go +++ b/management/internals/modules/agentnetwork/types/settings.go @@ -21,8 +21,7 @@ type Settings struct { AccountID string `gorm:"primaryKey"` // Domain is the gateway endpoint hostname agents call. Globally unique - // across accounts. Sized explicitly because MySQL cannot index an - // unbounded TEXT column; 255 covers the RFC 1035 253-octet bound. + // across accounts. 255 covers the RFC 1035 253-octet bound. Domain string `gorm:"type:varchar(255);uniqueIndex:idx_agent_network_settings_domain"` // ProxyAddress is the declared cluster address of the proxy serving this diff --git a/management/internals/modules/agentnetwork/types/usageoverview.go b/management/internals/modules/agentnetwork/types/usageoverview.go index 81e02c6b8..d46aec9c5 100644 --- a/management/internals/modules/agentnetwork/types/usageoverview.go +++ b/management/internals/modules/agentnetwork/types/usageoverview.go @@ -95,7 +95,7 @@ func bucketStart(t time.Time, g UsageGranularity) time.Time { // AggregateUsageByGranularity buckets the usage rows by the requested // granularity and returns the buckets ordered oldest-first. Aggregation is done // in Go (rather than per-engine SQL date_trunc) so granularities stay portable -// across SQLite/Postgres/MySQL and easy to extend. +// across SQLite and Postgres and easy to extend. func AggregateUsageByGranularity(rows []*AgentNetworkUsage, g UsageGranularity) []*AgentNetworkUsageBucket { byPeriod := make(map[string]*AgentNetworkUsageBucket) for _, r := range rows { diff --git a/management/internals/shared/db/engine.go b/management/internals/shared/db/engine.go index 02a3f5570..6cf03e239 100644 --- a/management/internals/shared/db/engine.go +++ b/management/internals/shared/db/engine.go @@ -6,5 +6,4 @@ type Engine string const ( SqliteStoreEngine Engine = "sqlite" PostgresStoreEngine Engine = "postgres" - MysqlStoreEngine Engine = "mysql" ) diff --git a/management/internals/shared/db/open.go b/management/internals/shared/db/open.go index 48a7bf330..c573e8c08 100644 --- a/management/internals/shared/db/open.go +++ b/management/internals/shared/db/open.go @@ -11,7 +11,6 @@ import ( "time" "github.com/jackc/pgx/v5/pgxpool" - "gorm.io/driver/mysql" "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" @@ -121,30 +120,6 @@ func OpenPostgres(ctx context.Context, dsn string, pool PoolConfig) (*Conn, erro return conn, nil } -// MysqlDSN adds the connection parameters every MySQL handle needs, keeping -// the options already present in dsn. -func MysqlDSN(dsn string) string { - separator := "?" - if strings.Contains(dsn, "?") { - separator = "&" - } - return dsn + separator + "charset=utf8&parseTime=True&loc=Local" -} - -// OpenMysql opens a MySQL database through gorm. -func OpenMysql(ctx context.Context, dsn string) (*Conn, error) { - gormDB, err := gorm.Open(mysql.Open(MysqlDSN(dsn)), GormConfig()) - if err != nil { - return nil, err - } - conn, err := NewConn(ctx, gormDB, MysqlStoreEngine, nil) - if err != nil { - closeGorm(gormDB) - return nil, err - } - return conn, nil -} - func newPgxPool(ctx context.Context, dsn string, cfg PoolConfig) (*pgxpool.Pool, error) { config, err := pgxpool.ParseConfig(dsn) if err != nil { diff --git a/management/internals/shared/db/open_test.go b/management/internals/shared/db/open_test.go deleted file mode 100644 index e6b506b78..000000000 --- a/management/internals/shared/db/open_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package db - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestMysqlDSN(t *testing.T) { - assert.Equal(t, "user:pw@tcp(host:3306)/db?charset=utf8&parseTime=True&loc=Local", MysqlDSN("user:pw@tcp(host:3306)/db")) - assert.Equal(t, "user:pw@tcp(host:3306)/db?tls=true&charset=utf8&parseTime=True&loc=Local", MysqlDSN("user:pw@tcp(host:3306)/db?tls=true")) -} diff --git a/management/internals/shared/db/transaction.go b/management/internals/shared/db/transaction.go index 9699aaee6..3930033d5 100644 --- a/management/internals/shared/db/transaction.go +++ b/management/internals/shared/db/transaction.go @@ -39,10 +39,7 @@ func (c *Conn) RunInTx(ctx context.Context, fn func(tx *Tx) error) error { return err } - err := c.withForeignKeyChecksDisabled(tx, func() error { - return fn(&Tx{db: tx}) - }) - if err != nil { + if err := fn(&Tx{db: tx}); err != nil { tx.Rollback() c.logIfTimedOut(ctx, timeoutCtx, err, "transaction", startTime) return err @@ -73,31 +70,6 @@ func (c *Conn) applyStatementTimeouts(tx *gorm.DB) error { return nil } -// withForeignKeyChecksDisabled runs fn with MySQL's FK checks off, which avoids -// deadlocks on MySQL and Aurora without needing SUPER privilege. The setting is -// session-scoped and survives a rollback, so it is turned back on whenever fn -// returns or panics; otherwise the pooled connection would keep it disabled. -func (c *Conn) withForeignKeyChecksDisabled(tx *gorm.DB, fn func() error) (err error) { - if c.engine != MysqlStoreEngine { - return fn() - } - if err := tx.Exec("SET FOREIGN_KEY_CHECKS = 0").Error; err != nil { - return fmt.Errorf("failed to disable FK checks: %w", err) - } - defer func() { - restoreErr := tx.Exec("SET FOREIGN_KEY_CHECKS = 1").Error - if restoreErr == nil { - return - } - if err == nil { - err = fmt.Errorf("failed to re-enable FK checks: %w", restoreErr) - return - } - log.WithContext(tx.Statement.Context).Warnf("failed to re-enable FK checks after failed transaction: %v", restoreErr) - }() - return fn() -} - func (c *Conn) logIfTimedOut(ctx, timeoutCtx context.Context, err error, phase string, startTime time.Time) { if errors.Is(err, context.DeadlineExceeded) || errors.Is(timeoutCtx.Err(), context.DeadlineExceeded) { log.WithContext(ctx).Warnf("%s exceeded %s timeout after %v, stack: %s", phase, c.txTimeout, time.Since(startTime), debug.Stack()) diff --git a/management/server/account.go b/management/server/account.go index 59991a4e4..68150a947 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -152,7 +152,6 @@ func (am *DefaultAccountManager) runAccountDeletionHooks(ctx context.Context, ac func isUniqueConstraintError(err error) bool { switch { case strings.Contains(err.Error(), "(SQLSTATE 23505)"), - strings.Contains(err.Error(), "Error 1062 (23000)"), strings.Contains(err.Error(), "UNIQUE constraint failed"): return true diff --git a/management/server/account_test.go b/management/server/account_test.go index ef7525961..8f7b1f18b 100644 --- a/management/server/account_test.go +++ b/management/server/account_test.go @@ -3759,7 +3759,7 @@ func setupNetworkMapTest(t *testing.T) (*DefaultAccountManager, *update_channel. // peerUpdateTimeout bounds how long peerShouldReceiveUpdate and its outer // wrappers wait for an expected update message. Sized for slow CI runners -// (MySQL, FreeBSD, loaded sqlite) where the channel publish can take +// (FreeBSD, loaded sqlite) where the channel publish can take // seconds. Only runs down on failure; passing tests return immediately // when the channel delivers. const peerUpdateTimeout = 5 * time.Second diff --git a/management/server/management_proto_test.go b/management/server/management_proto_test.go index 4e21ea7a3..9ca94d2e6 100644 --- a/management/server/management_proto_test.go +++ b/management/server/management_proto_test.go @@ -429,10 +429,6 @@ func Test_SyncStatusRace(t *testing.T) { if os.Getenv("NETBIRD_STORE_ENGINE") == "postgres" { t.Skip("Skipping on CI and Postgres store") } - - if os.Getenv("NETBIRD_STORE_ENGINE") == "mysql" { - t.Skip("Skipping on CI and MySQL store") - } } for i := 0; i < 500; i++ { t.Run(fmt.Sprintf("TestRun-%d", i), func(t *testing.T) { diff --git a/management/server/migration/migration.go b/management/server/migration/migration.go index 6d8ed90cc..239166c99 100644 --- a/management/server/migration/migration.go +++ b/management/server/migration/migration.go @@ -19,19 +19,11 @@ import ( "gorm.io/gorm/clause" ) -func GetColumnName(db *gorm.DB, column string) string { - if db.Name() == "mysql" { - return fmt.Sprintf("`%s`", column) - } - return column -} - // MigrateFieldFromGobToJSON migrates a column from Gob encoding to JSON encoding. // T is the type of the model that contains the field to be migrated. // S is the type of the field to be migrated. func MigrateFieldFromGobToJSON[T any, S any](ctx context.Context, db *gorm.DB, fieldName string) error { - orgColumnName := fieldName - oldColumnName := GetColumnName(db, orgColumnName) + oldColumnName := fieldName newColumnName := fieldName + "_tmp" var model T @@ -86,7 +78,7 @@ func MigrateFieldFromGobToJSON[T any, S any](ctx context.Context, db *gorm.DB, f for _, row := range rows { var field S - str, ok := row[orgColumnName].(string) + str, ok := row[oldColumnName].(string) if !ok { return fmt.Errorf("type assertion failed") } @@ -125,8 +117,7 @@ func MigrateFieldFromGobToJSON[T any, S any](ctx context.Context, db *gorm.DB, f // MigrateNetIPFieldFromBlobToJSON migrates a Net IP column from Blob encoding to JSON encoding. // T is the type of the model that contains the field to be migrated. func MigrateNetIPFieldFromBlobToJSON[T any](ctx context.Context, db *gorm.DB, fieldName string, indexName string) error { - orgColumnName := fieldName - oldColumnName := GetColumnName(db, orgColumnName) + oldColumnName := fieldName newColumnName := fieldName + "_tmp" var model T @@ -178,7 +169,7 @@ func MigrateNetIPFieldFromBlobToJSON[T any](ctx context.Context, db *gorm.DB, fi for _, row := range rows { var blobValue string - if columnValue := row[orgColumnName]; columnValue != nil { + if columnValue := row[oldColumnName]; columnValue != nil { value, ok := columnValue.(string) if !ok { return fmt.Errorf("type assertion failed") @@ -225,8 +216,7 @@ func MigrateNetIPFieldFromBlobToJSON[T any](ctx context.Context, db *gorm.DB, fi } func MigrateSetupKeyToHashedSetupKey[T any](ctx context.Context, db *gorm.DB) error { - orgColumnName := "key" - oldColumnName := GetColumnName(db, orgColumnName) + oldColumnName := "key" newColumnName := "key_secret" var model T @@ -268,7 +258,7 @@ func MigrateSetupKeyToHashedSetupKey[T any](ctx context.Context, db *gorm.DB) er for _, row := range rows { var plainKey string - if columnValue := row[orgColumnName]; columnValue != nil { + if columnValue := row[oldColumnName]; columnValue != nil { value, ok := columnValue.(string) if !ok { return fmt.Errorf("type assertion failed") @@ -394,33 +384,13 @@ func CreateIndexIfNotExists[T any](ctx context.Context, db *gorm.DB, indexName s return fmt.Errorf("failed to parse model schema: %w", err) } tableName := stmt.Schema.Table - dialect := db.Name() if db.Migrator().HasIndex(&model, indexName) { log.WithContext(ctx).Infof("index %s already exists on table %s", indexName, tableName) return nil } - var columnClause string - if dialect == "mysql" { - var withLength []string - for _, col := range columns { - quotedCol := fmt.Sprintf("`%s`", col) - if col == "ip" || col == "dns_label" || col == "key" { - withLength = append(withLength, fmt.Sprintf("%s(64)", quotedCol)) - } else { - withLength = append(withLength, quotedCol) - } - } - columnClause = strings.Join(withLength, ", ") - } else { - columnClause = strings.Join(columns, ", ") - } - - createStmt := fmt.Sprintf("CREATE UNIQUE INDEX %s ON %s (%s)", indexName, tableName, columnClause) - if dialect == "postgres" || dialect == "sqlite" { - createStmt = strings.Replace(createStmt, "CREATE UNIQUE INDEX", "CREATE UNIQUE INDEX IF NOT EXISTS", 1) - } + createStmt := fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (%s)", indexName, tableName, strings.Join(columns, ", ")) log.WithContext(ctx).Infof("executing index creation: %s", createStmt) if err := db.Exec(createStmt).Error; err != nil { @@ -505,14 +475,6 @@ func hasForeignKey(db *gorm.DB, table, column string) bool { AND kcu.table_name = ? AND kcu.column_name = ? `, table, column).Scan(&count) - case "mysql": - db.Raw(` - SELECT COUNT(*) FROM information_schema.key_column_usage - WHERE table_schema = DATABASE() - AND table_name = ? - AND column_name = ? - AND referenced_table_name IS NOT NULL - `, table, column).Scan(&count) default: // sqlite type fkInfo struct { From string @@ -592,7 +554,7 @@ func RemoveDuplicatePeerKeys(ctx context.Context, db *gorm.DB) error { return nil } - keyColumn := GetColumnName(db, "key") + keyColumn := "key" var duplicates []struct { Key string diff --git a/management/server/migration/migration_agentnetwork.go b/management/server/migration/migration_agentnetwork.go index c6cda56c2..7e19c0298 100644 --- a/management/server/migration/migration_agentnetwork.go +++ b/management/server/migration/migration_agentnetwork.go @@ -33,15 +33,6 @@ func (agentNetworkSettingsMigration) TableName() string { return "agent_network_ // bootstrap always wrote both, so such a row indicates corruption and the // migration fails loudly rather than leaving an empty domain to collide with // the unique index confusingly. -// -// The transaction is real only on sqlite and postgres, where DDL is -// transactional. MySQL implicitly commits around every ALTER TABLE, so there -// each step stands alone; what makes an interrupted run resumable on MySQL is -// that every step is guarded by the schema state it changes — the entry check -// fires while either legacy column remains, the adds skip existing columns, -// the backfill and its loud-failure check run only while the legacy cluster -// column exists (they provably completed before any drop), and each drop -// skips what is already gone. func MigrateAgentNetworkSettingsToDomain(ctx context.Context, db *gorm.DB) error { model := &agentNetworkSettingsMigration{} migrator := db.Migrator() @@ -66,14 +57,9 @@ func MigrateAgentNetworkSettingsToDomain(ctx context.Context, db *gorm.DB) error } if hasCluster { - concat := "subdomain || '.' || cluster" - if tx.Name() == "mysql" { - concat = "CONCAT(subdomain, '.', cluster)" - } - res := tx.Exec(fmt.Sprintf( - "UPDATE agent_network_settings SET domain = %s, proxy_address = cluster WHERE (domain IS NULL OR domain = '') AND cluster <> '' AND subdomain <> ''", - concat, - )) + res := tx.Exec( + "UPDATE agent_network_settings SET domain = subdomain || '.' || cluster, proxy_address = cluster WHERE (domain IS NULL OR domain = '') AND cluster <> '' AND subdomain <> ''", + ) if res.Error != nil { return fmt.Errorf("backfill agent_network_settings domain: %w", res.Error) } diff --git a/management/server/migration/migration_test.go b/management/server/migration/migration_test.go index 868332fdf..54c587a8a 100644 --- a/management/server/migration/migration_test.go +++ b/management/server/migration/migration_test.go @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gorm.io/driver/mysql" "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" @@ -32,17 +31,6 @@ func setupDatabase(t *testing.T) *gorm.DB { var dsn string var cleanup func() switch os.Getenv("NETBIRD_STORE_ENGINE") { - case "mysql": - cleanup, dsn, err = testutil.CreateMysqlTestContainer() - if err != nil { - t.Fatalf("Failed to create MySQL test container: %v", err) - } - - if dsn == "" { - t.Fatal("MySQL connection string is empty, ensure the test container is running") - } - - db, err = gorm.Open(mysql.Open(dsn+"?charset=utf8&parseTime=True&loc=Local"), &gorm.Config{}) case "postgres": cleanup, dsn, err = testutil.CreatePostgresTestContainer() if err != nil { @@ -597,9 +585,8 @@ func TestCleanupOrphanedResources_Idempotent(t *testing.T) { } func TestCleanupOrphanedResources_SkipsWhenForeignKeyExists(t *testing.T) { - engine := os.Getenv("NETBIRD_STORE_ENGINE") - if engine != "postgres" && engine != "mysql" { - t.Skip("FK constraint early-exit test requires postgres or mysql") + if os.Getenv("NETBIRD_STORE_ENGINE") != "postgres" { + t.Skip("FK constraint early-exit test requires postgres") } db := setupDatabase(t) @@ -614,24 +601,12 @@ func TestCleanupOrphanedResources_SkipsWhenForeignKeyExists(t *testing.T) { require.NoError(t, db.Create(&testChildWithFK{ID: "c1", ParentID: "p1"}).Error) require.NoError(t, db.Create(&testChildWithFK{ID: "c2", ParentID: "p2"}).Error) - switch engine { - case "postgres": - require.NoError(t, db.Exec("ALTER TABLE test_children DROP CONSTRAINT fk_test_children_parent").Error) - require.NoError(t, db.Exec("DELETE FROM test_parents WHERE id = ?", "p2").Error) - require.NoError(t, db.Exec( - "ALTER TABLE test_children ADD CONSTRAINT fk_test_children_parent "+ - "FOREIGN KEY (parent_id) REFERENCES test_parents(id) NOT VALID", - ).Error) - case "mysql": - require.NoError(t, db.Exec("SET FOREIGN_KEY_CHECKS = 0").Error) - require.NoError(t, db.Exec("ALTER TABLE test_children DROP FOREIGN KEY fk_test_children_parent").Error) - require.NoError(t, db.Exec("DELETE FROM test_parents WHERE id = ?", "p2").Error) - require.NoError(t, db.Exec( - "ALTER TABLE test_children ADD CONSTRAINT fk_test_children_parent "+ - "FOREIGN KEY (parent_id) REFERENCES test_parents(id)", - ).Error) - require.NoError(t, db.Exec("SET FOREIGN_KEY_CHECKS = 1").Error) - } + require.NoError(t, db.Exec("ALTER TABLE test_children DROP CONSTRAINT fk_test_children_parent").Error) + require.NoError(t, db.Exec("DELETE FROM test_parents WHERE id = ?", "p2").Error) + require.NoError(t, db.Exec( + "ALTER TABLE test_children ADD CONSTRAINT fk_test_children_parent "+ + "FOREIGN KEY (parent_id) REFERENCES test_parents(id) NOT VALID", + ).Error) err = migration.CleanupOrphanedResources[testChildWithFK, testParent](context.Background(), db, "parent_id") require.NoError(t, err) @@ -717,8 +692,6 @@ func TestFoldCostAggregatesIntoBuckets_SkipsAlreadyMigrated(t *testing.T) { require.NoError(t, db.Migrator().DropTable(&agentNetworkTypes.AgentNetworkUsage{})) require.NoError(t, db.AutoMigrate(&agentNetworkTypes.AgentNetworkUsage{})) - // Timestamp must be set explicitly: a zero time.Time serialises as - // '0000-00-00 00:00:00', which MySQL rejects under strict mode. require.NoError(t, db.Create(&agentNetworkTypes.AgentNetworkUsage{ ID: "u1", AccountID: "acct-1", Model: "claude-sonnet-4-6", Timestamp: time.Date(2026, 5, 5, 9, 0, 0, 0, time.UTC), @@ -819,10 +792,9 @@ func TestMigrateAgentNetworkSettingsToDomain_FailsOnUnmigratableRow(t *testing.T assert.Contains(t, err.Error(), "resolve them manually", "the error must tell the operator what to do") } -// partialAgentNetworkSettings models the one non-atomic state a MySQL run can -// be interrupted in: DDL auto-commits there, so a crash between the two legacy -// column drops leaves subdomain behind while cluster (and the completed -// backfill) are already committed. +// partialAgentNetworkSettings models a schema left between the two legacy +// column drops: subdomain is still present while cluster is gone and the +// backfill has already completed. type partialAgentNetworkSettings struct { AccountID string `gorm:"primaryKey"` Subdomain string @@ -832,8 +804,8 @@ type partialAgentNetworkSettings struct { func (partialAgentNetworkSettings) TableName() string { return "agent_network_settings" } -// TestMigrateAgentNetworkSettingsToDomain_ResumesAfterPartialDrop pins MySQL -// resumability: a rerun over the interrupted state must remove the leftover +// TestMigrateAgentNetworkSettingsToDomain_ResumesAfterPartialDrop pins +// resumability: a rerun over the partial state must remove the leftover // subdomain column without re-running the backfill (the cluster column that // feeds it is gone) and without touching the migrated values. func TestMigrateAgentNetworkSettingsToDomain_ResumesAfterPartialDrop(t *testing.T) { diff --git a/management/server/peer_test.go b/management/server/peer_test.go index 691630a3b..b75dd1b72 100644 --- a/management/server/peer_test.go +++ b/management/server/peer_test.go @@ -1566,8 +1566,7 @@ func Test_RegisterPeerBySetupKey(t *testing.T) { func Test_RegisterPeerRollbackOnFailure(t *testing.T) { engine := os.Getenv("NETBIRD_STORE_ENGINE") - if engine == "sqlite" || engine == "mysql" || engine == "" { - // we intentionally disabled foreign keys in mysql + if engine == "sqlite" || engine == "" { t.Skip("Skipping test because store is not respecting foreign keys") } if runtime.GOOS == "windows" { @@ -2219,10 +2218,6 @@ func Test_IsUniqueConstraintError(t *testing.T) { name: "PostgreSQL uniqueness error", engine: types.PostgresStoreEngine, }, - { - name: "MySQL uniqueness error", - engine: types.MysqlStoreEngine, - }, { name: "SQLite uniqueness error", engine: types.SqliteStoreEngine, diff --git a/management/server/store/sql_store.go b/management/server/store/sql_store.go index b705ba0a2..1734129c3 100644 --- a/management/server/store/sql_store.go +++ b/management/server/store/sql_store.go @@ -33,7 +33,6 @@ import ( const ( idQueryCondition = "id = ?" keyQueryCondition = "key = ?" - mysqlKeyQueryCondition = "`key` = ?" accountAndIDQueryCondition = "account_id = ? and id = ?" accountAndAnyIDQueryCondition = "account_id = ? and (id = ? or public_id = ?)" accountAndPeerIDQueryCondition = "account_id = ? and peer_id = ?" @@ -119,13 +118,6 @@ func (s *SqlStore) pgxPool() *pgxpool.Pool { return s.conn.Pool(s.tx) } -func GetKeyQueryCondition(s *SqlStore) string { - if s.conn.Engine() == db.MysqlStoreEngine { - return mysqlKeyQueryCondition - } - return keyQueryCondition -} - // AcquireGlobalLock acquires global lock across all the accounts and returns a function that releases the lock func (s *SqlStore) AcquireGlobalLock(ctx context.Context) (unlock func()) { log.WithContext(ctx).Tracef("acquiring global lock") @@ -174,15 +166,6 @@ func NewPostgresqlStore(ctx context.Context, dsn string, metrics telemetry.AppMe return newStore(ctx, conn, metrics, skipMigration) } -// NewMysqlStore creates a new MySQL store. -func NewMysqlStore(ctx context.Context, dsn string, metrics telemetry.AppMetrics, skipMigration bool) (*SqlStore, error) { - conn, err := db.OpenMysql(ctx, dsn) - if err != nil { - return nil, err - } - return newStore(ctx, conn, metrics, skipMigration) -} - // NewSqliteStoreFromFileStore restores a store from FileStore and stores SQLite DB in the file located in datadir. func NewSqliteStoreFromFileStore(ctx context.Context, fileStore *FileStore, dataDir string, metrics telemetry.AppMetrics, skipMigration bool) (*SqlStore, error) { store, err := NewSqliteStore(ctx, dataDir, metrics, skipMigration) @@ -240,11 +223,6 @@ func NewPostgresqlStoreForTests(ctx context.Context, dsn string, metrics telemet return newStore(ctx, conn, metrics, skipMigration) } -// NewMysqlStoreFromSqlStore restores a store from SqlStore and stores MySQL DB. -func NewMysqlStoreFromSqlStore(ctx context.Context, sqliteStore *SqlStore, dsn string, metrics telemetry.AppMetrics) (*SqlStore, error) { - return newMysqlStoreFromSqlStore(ctx, sqliteStore, dsn, metrics, false) -} - // seedFromSqliteStore copies the installation ID and the accounts of the // sqlite seed store into a freshly created engine store. func seedFromSqliteStore(ctx context.Context, store, sqliteStore *SqlStore) error { @@ -259,20 +237,6 @@ func seedFromSqliteStore(ctx context.Context, store, sqliteStore *SqlStore) erro return nil } -func newMysqlStoreFromSqlStore(ctx context.Context, sqliteStore *SqlStore, dsn string, metrics telemetry.AppMetrics, skipMigration bool) (*SqlStore, error) { - store, err := NewMysqlStore(ctx, dsn, metrics, skipMigration) - if err != nil { - return nil, err - } - - if err := seedFromSqliteStore(ctx, store, sqliteStore); err != nil { - _ = store.Close(ctx) - return nil, err - } - - return store, nil -} - // ExecuteInTransaction runs operation in a transaction. A store that is already // bound to one joins it instead of opening a second, independent transaction. func (s *SqlStore) ExecuteInTransaction(ctx context.Context, operation func(store Store) error) error { diff --git a/management/server/store/sql_store_account.go b/management/server/store/sql_store_account.go index 3780de77d..5c488ece8 100644 --- a/management/server/store/sql_store_account.go +++ b/management/server/store/sql_store_account.go @@ -1084,10 +1084,8 @@ func (s *SqlStore) SaveAccountSettings(ctx context.Context, accountID string, se return status.Errorf(status.Internal, "failed to save account settings to store") } - // MySQL reports RowsAffected=0 for no-op updates where values don't change, - // unlike SQLite/Postgres which report matched rows. Skip the check since the - // caller (UpdateAccountSettings) already verified the account exists via - // GetAccountSettings with LockingStrengthUpdate. + // No RowsAffected check: the caller (UpdateAccountSettings) already verified + // the account exists via GetAccountSettings with LockingStrengthUpdate. return nil } diff --git a/management/server/store/sql_store_group_test.go b/management/server/store/sql_store_group_test.go index 58291ab44..abb7fa8de 100644 --- a/management/server/store/sql_store_group_test.go +++ b/management/server/store/sql_store_group_test.go @@ -3,7 +3,6 @@ package store import ( "context" "fmt" - "os" "testing" "github.com/stretchr/testify/require" @@ -69,10 +68,6 @@ func TestSqlStore_GetGroupsByIDs(t *testing.T) { } func TestSqlStore_CreateGroup(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Log("Skipping MySQL test on CI") - } - t.Setenv("NETBIRD_STORE_ENGINE", string(types.MysqlStoreEngine)) store, cleanup, err := NewTestStoreFromSQL(context.Background(), "../testdata/extended-store.sql", t.TempDir()) t.Cleanup(cleanup) require.NoError(t, err) diff --git a/management/server/store/sql_store_idp_migration.go b/management/server/store/sql_store_idp_migration.go index 760d2967c..46ccd7603 100644 --- a/management/server/store/sql_store_idp_migration.go +++ b/management/server/store/sql_store_idp_migration.go @@ -55,7 +55,6 @@ func (s *SqlStore) ListUsers(ctx context.Context) ([]*types.User, error) { } // txDeferFKConstraints defers foreign key constraint checks for the duration of the transaction. -// MySQL is already handled by s.transaction (SET FOREIGN_KEY_CHECKS = 0). func (s *SqlStore) txDeferFKConstraints(tx *gorm.DB) error { if s.conn.Engine() == types.SqliteStoreEngine { return tx.Exec("PRAGMA defer_foreign_keys = ON").Error diff --git a/management/server/store/sql_store_peer.go b/management/server/store/sql_store_peer.go index 657d2a6e0..77642eea7 100644 --- a/management/server/store/sql_store_peer.go +++ b/management/server/store/sql_store_peer.go @@ -375,7 +375,7 @@ func (s *SqlStore) GetAccountByPeerID(ctx context.Context, peerID string) (*type func (s *SqlStore) GetAccountByPeerPubKey(ctx context.Context, peerKey string) (*types.Account, error) { var peer nbpeer.Peer - result := s.db.Select("account_id").Take(&peer, GetKeyQueryCondition(s), peerKey) + result := s.db.Select("account_id").Take(&peer, keyQueryCondition, peerKey) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { @@ -394,7 +394,7 @@ func (s *SqlStore) GetAccountByPeerPubKey(ctx context.Context, peerKey string) ( func (s *SqlStore) GetAccountIDByPeerPubKey(ctx context.Context, peerKey string) (string, error) { var peer nbpeer.Peer var accountID string - result := s.db.Model(&peer).Select("account_id").Where(GetKeyQueryCondition(s), peerKey).Take(&accountID) + result := s.db.Model(&peer).Select("account_id").Where(keyQueryCondition, peerKey).Take(&accountID) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { return "", status.Errorf(status.NotFound, "account not found: index lookup failed") @@ -483,7 +483,7 @@ func (s *SqlStore) GetPeerByPeerPubKey(ctx context.Context, lockStrength Locking } var peer nbpeer.Peer - result := tx.Take(&peer, GetKeyQueryCondition(s), peerKey) + result := tx.Take(&peer, keyQueryCondition, peerKey) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { @@ -914,7 +914,7 @@ func (s *SqlStore) GetUserIDByPeerKey(ctx context.Context, lockStrength LockingS var userID string result := tx.Model(&nbpeer.Peer{}). Select("user_id"). - Take(&userID, GetKeyQueryCondition(s), peerKey) + Take(&userID, keyQueryCondition, peerKey) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { @@ -935,7 +935,7 @@ func (s *SqlStore) GetPeerIDByKey(ctx context.Context, lockStrength LockingStren var peerID string result := tx.Model(&nbpeer.Peer{}). Select("id"). - Where(GetKeyQueryCondition(s), key). + Where(keyQueryCondition, key). Limit(1). Scan(&peerID) if result.Error != nil { diff --git a/management/server/store/sql_store_pgx_parity_test.go b/management/server/store/sql_store_pgx_parity_test.go index 1f17817d0..4af0b5540 100644 --- a/management/server/store/sql_store_pgx_parity_test.go +++ b/management/server/store/sql_store_pgx_parity_test.go @@ -13,7 +13,7 @@ import ( ) // TestPgxServiceColumnsMatchGorm guards the Postgres pgx read path against -// drifting from the gorm model. The SQLite/MySQL gorm path loads rows by struct, +// drifting from the gorm model. The SQLite gorm path loads rows by struct, // so a new column on a model is picked up automatically, but the hand-written // pgx SELECT in sql_store.go must be updated by hand. This test fails when a // gorm column is missing from the pgx column list, which otherwise silently diff --git a/management/server/store/sql_store_proxy.go b/management/server/store/sql_store_proxy.go index bdccd282c..c6a08b508 100644 --- a/management/server/store/sql_store_proxy.go +++ b/management/server/store/sql_store_proxy.go @@ -21,8 +21,8 @@ func (s *SqlStore) GetProxyMetrics(ctx context.Context) (ProxyMetrics, error) { var m ProxyMetrics activeCutoff := time.Now().Add(-proxyActiveThreshold) - // COUNT(DISTINCT ... CASE WHEN ...) is portable across sqlite/postgres - // (MySQL too) and keeps the round-trip to one. proxy.StatusConnected + // COUNT(DISTINCT ... CASE WHEN ...) is portable across sqlite and postgres + // and keeps the round-trip to one. proxy.StatusConnected // is the same string the cluster-capability queries use; the active // window matches the cluster-capability semantics (only proxies // heartbeating within ~2 * heartbeat interval count as connected). @@ -292,7 +292,7 @@ func (s *SqlStore) GetProxyClusters(ctx context.Context, accountID string) ([]pr "cluster_address AS address, "+ // COUNT(CASE WHEN ... THEN 1 END) counts only non-NULL — i.e. only // rows that satisfy the predicate — so it works portably across - // sqlite/postgres/mysql without dialect-specific FILTER syntax. + // sqlite and postgres without dialect-specific FILTER syntax. "COUNT(CASE WHEN status = ? AND last_seen > ? THEN 1 END) AS connected_proxies, "+ // MAX(CASE …) > 0 expresses BOOL_OR in a way Postgres tolerates // (Postgres can't MAX a boolean column). diff --git a/management/server/store/sql_store_setup_key.go b/management/server/store/sql_store_setup_key.go index ac42ef725..ef63b2fca 100644 --- a/management/server/store/sql_store_setup_key.go +++ b/management/server/store/sql_store_setup_key.go @@ -18,7 +18,7 @@ import ( func (s *SqlStore) GetAccountBySetupKey(ctx context.Context, setupKey string) (*types.Account, error) { var key types.SetupKey - result := s.db.Select("account_id").Take(&key, GetKeyQueryCondition(s), setupKey) + result := s.db.Select("account_id").Take(&key, keyQueryCondition, setupKey) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { return nil, status.NewSetupKeyNotFoundError(setupKey) @@ -99,7 +99,7 @@ func (s *SqlStore) getSetupKeys(ctx context.Context, accountID string) ([]types. func (s *SqlStore) GetAccountIDBySetupKey(ctx context.Context, setupKey string) (string, error) { var accountID string - result := s.db.Model(&types.SetupKey{}).Select("account_id").Where(GetKeyQueryCondition(s), setupKey).Take(&accountID) + result := s.db.Model(&types.SetupKey{}).Select("account_id").Where(keyQueryCondition, setupKey).Take(&accountID) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { return "", status.NewSetupKeyNotFoundError(setupKey) @@ -123,7 +123,7 @@ func (s *SqlStore) GetSetupKeyBySecret(ctx context.Context, lockStrength Locking var setupKey types.SetupKey result := tx. - Take(&setupKey, GetKeyQueryCondition(s), key) + Take(&setupKey, keyQueryCondition, key) if result.Error != nil { if errors.Is(result.Error, gorm.ErrRecordNotFound) { diff --git a/management/server/store/sql_store_test.go b/management/server/store/sql_store_test.go index 24f9a410d..867657d8b 100644 --- a/management/server/store/sql_store_test.go +++ b/management/server/store/sql_store_test.go @@ -164,7 +164,6 @@ func TestMigrate(t *testing.T) { err = migratePreAuto(context.Background(), store.(*SqlStore).db) require.NoError(t, err, "Migration should not fail on migrated db") - } func newAccount(store Store, id int) error { @@ -352,10 +351,6 @@ func TestSqlStore_DatabaseBlocking(t *testing.T) { } func TestSqlStore_ExecuteInTransaction_Timeout(t *testing.T) { - if os.Getenv("NETBIRD_STORE_ENGINE") == "mysql" { - t.Skip("Skipping timeout test for MySQL") - } - t.Setenv("NB_STORE_TRANSACTION_TIMEOUT", "1s") store, cleanup, err := NewTestStoreFromSQL(context.Background(), "", t.TempDir()) @@ -429,49 +424,6 @@ func TestNewSqliteStore_BusyTimeoutRespectsUserOverride(t *testing.T) { } } -func TestSqlStore_ExecuteInTransaction_RestoresForeignKeyChecksOnMysql(t *testing.T) { - runTestForAllEngines(t, "", func(t *testing.T, store Store) { - sqlStore := store.(*SqlStore) - if sqlStore.conn.Engine() != types.MysqlStoreEngine { - t.Skip("FOREIGN_KEY_CHECKS is MySQL specific") - } - sqlDB, err := sqlStore.GetDB().DB() - require.NoError(t, err) - sqlDB.SetMaxOpenConns(1) - ctx := context.Background() - - foreignKeyChecks := func() int { - var enabled int - require.NoError(t, sqlStore.GetDB().Raw("SELECT @@SESSION.foreign_key_checks").Scan(&enabled).Error) - return enabled - } - - err = store.ExecuteInTransaction(ctx, func(Store) error { return assert.AnError }) - require.ErrorIs(t, err, assert.AnError) - assert.Equal(t, 1, foreignKeyChecks()) - - require.Panics(t, func() { - _ = store.ExecuteInTransaction(ctx, func(Store) error { panic("boom") }) - }) - assert.Equal(t, 1, foreignKeyChecks()) - - err = sqlStore.transaction(ctx, func(*gorm.DB) error { return assert.AnError }) - require.ErrorIs(t, err, assert.AnError) - assert.Equal(t, 1, foreignKeyChecks()) - - err = store.ExecuteInTransaction(ctx, func(transaction Store) error { - bound := transaction.(*SqlStore) - require.NoError(t, bound.transaction(ctx, func(*gorm.DB) error { return nil })) - var enabled int - require.NoError(t, bound.GetDB().Raw("SELECT @@SESSION.foreign_key_checks").Scan(&enabled).Error) - assert.Equal(t, 0, enabled, "a savepoint must not re-enable FK checks for the rest of the transaction") - return nil - }) - require.NoError(t, err) - assert.Equal(t, 1, foreignKeyChecks()) - }) -} - func TestSqlStore_Transaction_RollsBackOnError(t *testing.T) { runTestForAllEngines(t, "../testdata/extended-store.sql", func(t *testing.T, store Store) { ctx := context.Background() diff --git a/management/server/store/store.go b/management/server/store/store.go index 174e97b48..6d20c1e2b 100644 --- a/management/server/store/store.go +++ b/management/server/store/store.go @@ -4,7 +4,6 @@ package store import ( "context" - "database/sql" "errors" "fmt" "net" @@ -20,7 +19,6 @@ import ( "github.com/google/uuid" log "github.com/sirupsen/logrus" - "gorm.io/driver/mysql" "gorm.io/driver/postgres" "gorm.io/gorm" "gorm.io/gorm/logger" @@ -452,8 +450,6 @@ type AgentNetworkMetrics struct { const ( PostgresDsnEnv = "NB_STORE_ENGINE_POSTGRES_DSN" PostgresDsnEnvLegacy = "NETBIRD_STORE_ENGINE_POSTGRES_DSN" - mysqlDsnEnv = "NB_STORE_ENGINE_MYSQL_DSN" - mysqlDsnEnvLegacy = "NETBIRD_STORE_ENGINE_MYSQL_DSN" ) // lookupDSNEnv checks the NB_ env var first, then falls back to the legacy NETBIRD_ env var. @@ -464,7 +460,7 @@ func lookupDSNEnv(nbKey, legacyKey string) (string, bool) { return os.LookupEnv(legacyKey) } -var supportedEngines = []types.Engine{types.SqliteStoreEngine, types.PostgresStoreEngine, types.MysqlStoreEngine} +var supportedEngines = []types.Engine{types.SqliteStoreEngine, types.PostgresStoreEngine} func getStoreEngineFromEnv() types.Engine { // NETBIRD_STORE_ENGINE supposed to be used in tests. Otherwise, rely on the config file. @@ -474,7 +470,7 @@ func getStoreEngineFromEnv() types.Engine { } value := types.Engine(strings.ToLower(kind)) - if slices.Contains(supportedEngines, value) { + if slices.Contains(supportedEngines, value) || value == types.MysqlStoreEngine { return value } @@ -540,12 +536,8 @@ func OpenConn(ctx context.Context, kind types.Engine, dataDir string) (*db.Conn, } return db.OpenPostgres(ctx, dsn, db.DefaultPoolConfig) case types.MysqlStoreEngine: - log.WithContext(ctx).Info("using MySQL store engine") - dsn, ok := lookupDSNEnv(mysqlDsnEnv, mysqlDsnEnvLegacy) - if !ok { - return nil, fmt.Errorf("%s is not set", mysqlDsnEnv) - } - return db.OpenMysql(ctx, dsn) + return nil, fmt.Errorf("the MySQL store engine is no longer supported. Please refer to the documentation for " + + "migrating to PostgreSQL: https://docs.netbird.io/selfhosted/migration/mysql-to-postgresql") default: return nil, fmt.Errorf("unsupported kind of store: %s", kind) } @@ -725,7 +717,7 @@ func NewTestStoreFromSQL(ctx context.Context, filename string, dataDir string) ( } if filename != "" { - err = LoadSQL(conn.DB(nil), filename, translaterForSql(kind)) + err = LoadSQL(conn.DB(nil), filename) if err != nil { _ = conn.Close() return nil, nil, fmt.Errorf("failed to load SQL file: %v", err) @@ -761,6 +753,7 @@ func NewTestStoreFromSQL(ctx context.Context, filename string, dataDir string) ( return nil, nil, fmt.Errorf("failed to create test store after %d attempts: %v", maxRetries, err) } +// NewTestStore creates a test store and optionally loads SQL after migrating its schema. func NewTestStore(ctx context.Context, filename string, dataDir string) (Store, func(), error) { kind := getStoreEngineFromEnv() if kind == "" { @@ -786,7 +779,7 @@ func NewTestStore(ctx context.Context, filename string, dataDir string) (Store, } if filename != "" { - err = LoadSQL(sqlStore.db, filename, translaterForSql(kind)) + err = LoadSQL(sqlStore.db, filename) if err != nil { _ = sqlStore.Close(ctx) return nil, nil, fmt.Errorf("failed to load SQL file: %v", err) @@ -802,18 +795,6 @@ func NewTestStore(ctx context.Context, filename string, dataDir string) (Store, return sqlStore, cleanup, nil } -// use to translate double-quote escaped reserved names (ansi) -// to mysql tick-escaped ones -func translaterForSql(kind types.Engine) *strings.Replacer { - if kind != types.MysqlStoreEngine { - return strings.NewReplacer() - } - return strings.NewReplacer( - "\"key\"", "`key`", - "\"groups\"", "`groups`", - ) -} - func addAllGroupToAccount(ctx context.Context, store Store) error { allAccounts := store.GetAllAccounts(ctx) for _, account := range allAccounts { @@ -843,9 +824,7 @@ func getSqlStoreEngine(ctx context.Context, sqliteStore *SqlStore, kind types.En var err error switch kind { case types.PostgresStoreEngine: - store, cleanup, err = newReusedPostgresStore(ctx, sqliteStore, kind) - case types.MysqlStoreEngine: - store, cleanup, err = newReusedMysqlStore(ctx, sqliteStore, kind) + store, cleanup, err = newReusedPostgresStore(ctx, sqliteStore) default: cleanup = func() { // sqlite doesn't need to be cleaned up @@ -877,7 +856,7 @@ func getSqlStoreEngine(ctx context.Context, sqliteStore *SqlStore, kind types.En return store, closeConnection, nil } -func newReusedPostgresStore(ctx context.Context, sqliteStore *SqlStore, kind types.Engine) (*SqlStore, func(), error) { +func newReusedPostgresStore(ctx context.Context, sqliteStore *SqlStore) (*SqlStore, func(), error) { dsn, ok := lookupDSNEnv(PostgresDsnEnv, PostgresDsnEnvLegacy) if !ok || dsn == "" { var err error @@ -891,7 +870,7 @@ func newReusedPostgresStore(ctx context.Context, sqliteStore *SqlStore, kind typ return nil, nil, fmt.Errorf("%s is not set", PostgresDsnEnv) } - db, err := openDBWithRetry(dsn, kind, 5) + db, err := openDBWithRetry(dsn, 5) if err != nil { return nil, nil, fmt.Errorf("failed to open postgres connection: %v", err) } @@ -902,7 +881,7 @@ func newReusedPostgresStore(ctx context.Context, sqliteStore *SqlStore, kind typ return nil, nil, err } - dsn, cleanup, err := createRandomDB(dsn, db, kind, template) + dsn, cleanup, err := createRandomDB(dsn, db, template) closeGormDB(db) @@ -928,74 +907,10 @@ func newReusedPostgresStore(ctx context.Context, sqliteStore *SqlStore, kind typ return store, cleanup, nil } -func newReusedMysqlStore(ctx context.Context, sqliteStore *SqlStore, kind types.Engine) (*SqlStore, func(), error) { - dsn, ok := lookupDSNEnv(mysqlDsnEnv, mysqlDsnEnvLegacy) - if !ok || dsn == "" { - var err error - _, dsn, err = testutil.CreateMysqlTestContainer() - if err != nil { - return nil, nil, err - } - } - - if dsn == "" { - return nil, nil, fmt.Errorf("%s is not set", mysqlDsnEnv) - } - - db, err := openDBWithRetry(dsn, kind, 5) - if err != nil { - return nil, nil, fmt.Errorf("failed to open mysql connection: %v", err) - } - - sqlDB, err := db.DB() - if err != nil { - return nil, nil, fmt.Errorf("failed to get underlying sql.DB: %v", err) - } - sqlDB.SetMaxOpenConns(1) - sqlDB.SetMaxIdleConns(1) - - tableDDL, err := mysqlSchemaTemplate(ctx, dsn, db) - if err != nil { - sqlDB.Close() - return nil, nil, err - } - - dsn, cleanup, err := createRandomDB(dsn, db, kind, "") - - sqlDB.Close() - - if err != nil { - return nil, nil, err - } - - if err := cloneMysqlSchema(ctx, dsn, tableDDL); err != nil { - cleanup() - return nil, nil, err - } - - var store *SqlStore - if sqliteStore != nil { - store, err = newMysqlStoreFromSqlStore(ctx, sqliteStore, dsn, nil, true) - if err != nil { - cleanup() - return nil, nil, err - } - } else { - store, err = NewMysqlStore(ctx, dsn, nil, false) - if err != nil { - cleanup() - return nil, nil, err - } - } - - return store, cleanup, nil -} - // schemaTemplates remembers, per engine and server, a database that went // through the full migration once in this process. Every later test database // is cloned from it, so a test pays for CREATE DATABASE and a schema copy -// instead of the 40-table AutoMigrate plus every pre and post migration, which -// is what made each MySQL test store cost well over a second in CI. +// instead of the 40-table AutoMigrate plus every pre and post migration. var ( schemaTemplatesMu sync.Mutex schemaTemplates = map[string]*schemaTemplate{} @@ -1003,10 +918,6 @@ var ( type schemaTemplate struct { dbName string - // tableDDL holds the CREATE TABLE statements of the template. MySQL has no - // server-side database template, so the schema is replayed statement by - // statement into each test database. - tableDDL []string } func schemaTemplateKey(engine types.Engine, dsn string) string { @@ -1046,112 +957,6 @@ func postgresSchemaTemplate(ctx context.Context, baseDSN string, admin *gorm.DB) return name, nil } -// mysqlSchemaTemplate returns the CREATE TABLE statements of a fully migrated -// database, migrating one on first use. -func mysqlSchemaTemplate(ctx context.Context, baseDSN string, admin *gorm.DB) ([]string, error) { - schemaTemplatesMu.Lock() - defer schemaTemplatesMu.Unlock() - - key := schemaTemplateKey(types.MysqlStoreEngine, baseDSN) - if tpl, ok := schemaTemplates[key]; ok { - return tpl.tableDDL, nil - } - - name := newTestDBName("test_template") - if err := admin.Exec(fmt.Sprintf("CREATE DATABASE %s", name)).Error; err != nil { - return nil, fmt.Errorf("create mysql template database: %w", err) - } - - tplStore, err := NewMysqlStore(ctx, replaceDBName(baseDSN, name), nil, false) - if err != nil { - dropDatabase(admin, name) - return nil, fmt.Errorf("migrate mysql template database: %w", err) - } - tableDDL, err := mysqlTableDDL(ctx, tplStore.db, name) - tplStore.Close(ctx) - if err != nil { - dropDatabase(admin, name) - return nil, err - } - - schemaTemplates[key] = &schemaTemplate{dbName: name, tableDDL: tableDDL} - return tableDDL, nil -} - -func mysqlTableDDL(ctx context.Context, db *gorm.DB, dbName string) ([]string, error) { - sqlDB, err := db.DB() - if err != nil { - return nil, err - } - - tables, err := mysqlTableNames(ctx, sqlDB, dbName) - if err != nil { - return nil, err - } - - tableDDL := make([]string, 0, len(tables)) - for _, table := range tables { - var name, createStmt string - row := sqlDB.QueryRowContext(ctx, fmt.Sprintf("SHOW CREATE TABLE %s.%s", dbName, table)) - if err := row.Scan(&name, &createStmt); err != nil { - return nil, fmt.Errorf("read create statement of %s: %w", table, err) - } - tableDDL = append(tableDDL, createStmt) - } - return tableDDL, nil -} - -func mysqlTableNames(ctx context.Context, sqlDB *sql.DB, dbName string) ([]string, error) { - rows, err := sqlDB.QueryContext(ctx, fmt.Sprintf("SHOW TABLES FROM %s", dbName)) - if err != nil { - return nil, fmt.Errorf("list template tables: %w", err) - } - defer rows.Close() - - var tables []string - for rows.Next() { - var table string - if err := rows.Scan(&table); err != nil { - return nil, fmt.Errorf("scan template table name: %w", err) - } - tables = append(tables, table) - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("list template tables: %w", err) - } - return tables, nil -} - -// cloneMysqlSchema replays the template's CREATE TABLE statements into the -// database the DSN points at. -func cloneMysqlSchema(ctx context.Context, dsn string, tableDDL []string) error { - gormDB, err := gorm.Open(mysql.Open(db.MysqlDSN(dsn)), db.GormConfig()) - if err != nil { - return fmt.Errorf("connect to test database: %w", err) - } - sqlDB, err := gormDB.DB() - if err != nil { - return err - } - defer sqlDB.Close() - - // The statements come out of SHOW TABLES in name order, not dependency - // order, and their foreign keys reference tables of the session's default - // database. Pin a single connection so the session setting below covers - // every statement, and connect straight to the new database so unqualified - // references land there. - sqlDB.SetMaxOpenConns(1) - if _, err := sqlDB.ExecContext(ctx, "SET FOREIGN_KEY_CHECKS = 0"); err != nil { - return fmt.Errorf("disable foreign key checks: %w", err) - } - for _, stmt := range tableDDL { - if _, err := sqlDB.ExecContext(ctx, stmt); err != nil { - return fmt.Errorf("replay table definition: %w", err) - } - } - return nil -} - // dropDatabase removes a template that never became usable, so a failed setup // does not leave it behind on a shared server. The server may still be tearing // down the sessions the failed migration held, so the drop retries while @@ -1168,18 +973,12 @@ func closeGormDB(db *gorm.DB) { } } -func openDBWithRetry(dsn string, engine types.Engine, maxRetries int) (*gorm.DB, error) { +func openDBWithRetry(dsn string, maxRetries int) (*gorm.DB, error) { var gormDB *gorm.DB var err error for i := range maxRetries { - switch engine { - case types.PostgresStoreEngine: - gormDB, err = gorm.Open(postgres.Open(dsn), &gorm.Config{}) - case types.MysqlStoreEngine: - gormDB, err = gorm.Open(mysql.Open(db.MysqlDSN(dsn)), &gorm.Config{}) - } - + gormDB, err = gorm.Open(postgres.Open(dsn), &gorm.Config{}) if err == nil { return gormDB, nil } @@ -1193,13 +992,13 @@ func openDBWithRetry(dsn string, engine types.Engine, maxRetries int) (*gorm.DB, return nil, err } -// createRandomDB creates a uniquely named database for one test. On postgres a -// non-empty template is copied server-side with CREATE DATABASE ... TEMPLATE. -func createRandomDB(dsn string, admin *gorm.DB, engine types.Engine, template string) (string, func(), error) { +// createRandomDB creates a uniquely named database for one test. A non-empty +// template is copied server-side with CREATE DATABASE ... TEMPLATE. +func createRandomDB(dsn string, admin *gorm.DB, template string) (string, func(), error) { dbName := newTestDBName("test_db") createStmt := fmt.Sprintf("CREATE DATABASE %s", dbName) - if template != "" && engine == types.PostgresStoreEngine { + if template != "" { createStmt = fmt.Sprintf("CREATE DATABASE %s TEMPLATE %s", dbName, template) } if err := execWithTemplateRetry(admin, createStmt); err != nil { @@ -1209,59 +1008,28 @@ func createRandomDB(dsn string, admin *gorm.DB, engine types.Engine, template st originalDSN := dsn cleanup := func() { - var dropDB *gorm.DB - var err error - - switch engine { - case types.PostgresStoreEngine: - dropDB, err = gorm.Open(postgres.Open(originalDSN), &gorm.Config{ - SkipDefaultTransaction: true, - PrepareStmt: false, - Logger: logger.Default.LogMode(logger.Info), - }) - if err != nil { - log.Errorf("failed to connect for dropping database %s: %v", dbName, err) - return - } - defer func() { - if sqlDB, _ := dropDB.DB(); sqlDB != nil { - sqlDB.Close() - } - }() - + dropDB, err := gorm.Open(postgres.Open(originalDSN), &gorm.Config{ + SkipDefaultTransaction: true, + PrepareStmt: false, + Logger: logger.Default.LogMode(logger.Info), + }) + if err != nil { + log.Errorf("failed to connect for dropping database %s: %v", dbName, err) + return + } + defer func() { if sqlDB, _ := dropDB.DB(); sqlDB != nil { - sqlDB.SetMaxOpenConns(1) - sqlDB.SetMaxIdleConns(0) - sqlDB.SetConnMaxLifetime(time.Second) + sqlDB.Close() } + }() - err = dropDB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s WITH (FORCE)", dbName)).Error - - case types.MysqlStoreEngine: - dropDB, err = gorm.Open(mysql.Open(db.MysqlDSN(originalDSN)), &gorm.Config{ - SkipDefaultTransaction: true, - PrepareStmt: false, - }) - if err != nil { - log.Errorf("failed to connect for dropping database %s: %v", dbName, err) - return - } - defer func() { - if sqlDB, _ := dropDB.DB(); sqlDB != nil { - sqlDB.Close() - } - }() - - if sqlDB, _ := dropDB.DB(); sqlDB != nil { - sqlDB.SetMaxOpenConns(1) - sqlDB.SetMaxIdleConns(0) - sqlDB.SetConnMaxLifetime(time.Second) - } - - err = dropDB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", dbName)).Error + if sqlDB, _ := dropDB.DB(); sqlDB != nil { + sqlDB.SetMaxOpenConns(1) + sqlDB.SetMaxIdleConns(0) + sqlDB.SetConnMaxLifetime(time.Second) } - if err != nil { + if err := dropDB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s WITH (FORCE)", dbName)).Error; err != nil { log.Errorf("failed to drop database %s: %v", dbName, err) } } @@ -1288,7 +1056,8 @@ func replaceDBName(dsn, newDBName string) string { return re.ReplaceAllString(dsn, `${pre}`+newDBName+`${post}`) } -func LoadSQL(db *gorm.DB, filepath string, translator *strings.Replacer) error { +// LoadSQL executes the SQL statements from the given file. +func LoadSQL(db *gorm.DB, filepath string) error { sqlContent, err := os.ReadFile(filepath) if err != nil { return err @@ -1297,7 +1066,7 @@ func LoadSQL(db *gorm.DB, filepath string, translator *strings.Replacer) error { queries := strings.Split(string(sqlContent), ";") for _, query := range queries { - query = translator.Replace(strings.TrimSpace(query)) + query = strings.TrimSpace(query) if query != "" { err := db.Exec(query).Error if err != nil { diff --git a/management/server/testutil/store.go b/management/server/testutil/store.go index 3854c1173..ddb905d3e 100644 --- a/management/server/testutil/store.go +++ b/management/server/testutil/store.go @@ -8,74 +8,12 @@ import ( log "github.com/sirupsen/logrus" "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/modules/mysql" "github.com/testcontainers/testcontainers-go/modules/postgres" testcontainersredis "github.com/testcontainers/testcontainers-go/modules/redis" "github.com/testcontainers/testcontainers-go/wait" ) -var ( - pgContainer *postgres.PostgresContainer - mysqlContainer *mysql.MySQLContainer -) - -// CreateMysqlTestContainer creates a new MySQL container for testing. -func CreateMysqlTestContainer() (func(), string, error) { - ctx := context.Background() - - if mysqlContainer != nil { - connStr, err := mysqlContainer.ConnectionString(ctx) - if err != nil { - return nil, "", err - } - return noOpCleanup, connStr, nil - } - - var err error - mysqlContainer, err = mysql.Run(ctx, - "mlsmaycon/warmed-mysql:8", - mysql.WithDatabase("testing"), - mysql.WithUsername("root"), - mysql.WithPassword("testing"), - // Every test creates and drops a database with about 40 tables, so with - // the server defaults the run is dominated by durability work: each - // CREATE TABLE fsyncs the redo log, the binary log and the doublewrite - // buffer. None of it protects anything in a container that is discarded - // after the run. Tables stay in per-table files on purpose: in the shared - // system tablespace the cost of every CREATE and DROP grew with the number - // of databases the run had already created. - testcontainers.WithCmd("mysqld", - "--innodb-flush-log-at-trx-commit=0", - "--innodb-doublewrite=OFF", - "--skip-log-bin", - ), - testcontainers.WithWaitStrategy( - wait.ForLog("/usr/sbin/mysqld: ready for connections"). - WithOccurrence(1).WithStartupTimeout(15*time.Second).WithPollInterval(100*time.Millisecond), - ), - ) - if err != nil { - return nil, "", err - } - - cleanup := func() { - if mysqlContainer != nil { - timeoutCtx, cancelFunc := context.WithTimeout(ctx, 1*time.Second) - defer cancelFunc() - if err = mysqlContainer.Terminate(timeoutCtx); err != nil { - log.WithContext(ctx).Warnf("failed to stop mysql container %s: %s", mysqlContainer.GetContainerID(), err) - } - mysqlContainer = nil // reset the container to allow recreation - } - } - - talksConn, err := mysqlContainer.ConnectionString(ctx) - if err != nil { - return nil, "", err - } - - return cleanup, talksConn, nil -} +var pgContainer *postgres.PostgresContainer // CreatePostgresTestContainer creates a new PostgreSQL container for testing. func CreatePostgresTestContainer() (func(), string, error) { diff --git a/management/server/testutil/store_ios.go b/management/server/testutil/store_ios.go index 9e3b5ce4a..2a484724b 100644 --- a/management/server/testutil/store_ios.go +++ b/management/server/testutil/store_ios.go @@ -8,12 +8,6 @@ func CreatePostgresTestContainer() (func(), string, error) { }, "", nil } -func CreateMysqlTestContainer() (func(), string, error) { - return func() { - // Empty function for MySQL - }, "", nil -} - func CreateRedisTestContainer() (func(), string, error) { return func() { // Empty function for Redis diff --git a/management/server/types/store.go b/management/server/types/store.go index a13d52f91..726ca4c89 100644 --- a/management/server/types/store.go +++ b/management/server/types/store.go @@ -8,5 +8,5 @@ const ( PostgresStoreEngine = db.PostgresStoreEngine FileStoreEngine Engine = "jsonfile" SqliteStoreEngine = db.SqliteStoreEngine - MysqlStoreEngine = db.MysqlStoreEngine + MysqlStoreEngine Engine = "mysql" ) diff --git a/shared/management/client/rest/accounts_test.go b/shared/management/client/rest/accounts_test.go index e44ada298..9de802000 100644 --- a/shared/management/client/rest/accounts_test.go +++ b/shared/management/client/rest/accounts_test.go @@ -168,7 +168,7 @@ func TestAccounts_Integration_Update(t *testing.T) { }) } -// Account deletion on MySQL and PostgreSQL databases causes unknown errors +// Account deletion on PostgreSQL databases causes unknown errors // func TestAccounts_Integration_Delete(t *testing.T) { // withBlackBoxServer(t, func(c *rest.Client) { // accounts, err := c.Accounts.List(context.Background()) diff --git a/tools/mysql-migrate/LICENSE b/tools/mysql-migrate/LICENSE deleted file mode 100644 index be3f7b28e..000000000 --- a/tools/mysql-migrate/LICENSE +++ /dev/null @@ -1,661 +0,0 @@ - GNU AFFERO GENERAL PUBLIC LICENSE - Version 3, 19 November 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU Affero General Public License is a free, copyleft license for -software and other kinds of works, specifically designed to ensure -cooperation with the community in the case of network server software. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -our General Public Licenses are intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - Developers that use our General Public Licenses protect your rights -with two steps: (1) assert copyright on the software, and (2) offer -you this License which gives you legal permission to copy, distribute -and/or modify the software. - - A secondary benefit of defending all users' freedom is that -improvements made in alternate versions of the program, if they -receive widespread use, become available for other developers to -incorporate. Many developers of free software are heartened and -encouraged by the resulting cooperation. However, in the case of -software used on network servers, this result may fail to come about. -The GNU General Public License permits making a modified version and -letting the public access it on a server without ever releasing its -source code to the public. - - The GNU Affero General Public License is designed specifically to -ensure that, in such cases, the modified source code becomes available -to the community. It requires the operator of a network server to -provide the source code of the modified version running there to the -users of that server. Therefore, public use of a modified version, on -a publicly accessible server, gives the public access to the source -code of the modified version. - - An older license, called the Affero General Public License and -published by Affero, was designed to accomplish similar goals. This is -a different license, not a version of the Affero GPL, but Affero has -released a new version of the Affero GPL which permits relicensing under -this license. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU Affero General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Remote Network Interaction; Use with the GNU General Public License. - - Notwithstanding any other provision of this License, if you modify the -Program, your modified version must prominently offer all users -interacting with it remotely through a computer network (if your version -supports such interaction) an opportunity to receive the Corresponding -Source of your version by providing access to the Corresponding Source -from a network server at no charge, through some standard or customary -means of facilitating copying of software. This Corresponding Source -shall include the Corresponding Source for any work covered by version 3 -of the GNU General Public License that is incorporated pursuant to the -following paragraph. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the work with which it is combined will remain governed by version -3 of the GNU General Public License. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU Affero General Public License from time to time. Such new versions -will be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU Affero General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU Affero General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU Affero General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If your software can interact with users remotely through a computer -network, you should also make sure that it provides a way for users to -get its source. For example, if your program is a web application, its -interface could display a "Source" link that leads users to an archive -of the code. There are many ways you could offer source, and different -solutions will be better for different programs; see section 13 for the -specific requirements. - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU AGPL, see -. diff --git a/tools/mysql-migrate/copy.go b/tools/mysql-migrate/copy.go deleted file mode 100644 index 85e0c5e91..000000000 --- a/tools/mysql-migrate/copy.go +++ /dev/null @@ -1,357 +0,0 @@ -package main - -import ( - "context" - "database/sql" - "fmt" - "math" - "slices" - "strconv" - "strings" - - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgtype" -) - -type dialect struct { - quote func(string) string - tablesQuery string - columnsQuery string -} - -var mysqlDialect = dialect{ - quote: func(name string) string { return "`" + name + "`" }, - tablesQuery: `SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() AND table_type = 'BASE TABLE'`, - columnsQuery: `SELECT column_name FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = ?`, -} - -var sqliteDialect = dialect{ - quote: func(name string) string { return `"` + name + `"` }, - tablesQuery: `SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'`, - columnsQuery: `SELECT name FROM pragma_table_info(?)`, -} - -type column struct { - name string - pgType string -} - -// copyStore copies the tables the source shares with the target schema; -// source-only tables are legacy and skipped. -func copyStore(ctx context.Context, src *sql.DB, d dialect, tx pgx.Tx, skip map[string]bool) (int, int64, error) { - srcTables, err := queryStrings(ctx, src, d.tablesQuery) - if err != nil { - return 0, 0, fmt.Errorf("list source tables: %w", err) - } - dstTables, err := pgStrings(ctx, tx, `SELECT table_name FROM information_schema.tables WHERE table_schema = current_schema() AND table_type = 'BASE TABLE'`) - if err != nil { - return 0, 0, fmt.Errorf("list target tables: %w", err) - } - - var tables []string - for _, t := range srcTables { - if !skip[t] && slices.Contains(dstTables, t) { - tables = append(tables, t) - } - } - if len(tables) == 0 { - return 0, 0, fmt.Errorf("no NetBird tables found in the source") - } - - ordered, err := parentsFirst(ctx, tx, tables) - if err != nil { - return 0, 0, err - } - - var total int64 - for _, table := range ordered { - n, err := copyTable(ctx, src, d, tx, table) - if err != nil { - return 0, 0, fmt.Errorf("table %s: %w", table, err) - } - total += n - } - - if err := resetSequences(ctx, tx, tables); err != nil { - return 0, 0, err - } - return len(tables), total, nil -} - -func copyTable(ctx context.Context, src *sql.DB, d dialect, tx pgx.Tx, table string) (int64, error) { - cols, err := targetColumns(ctx, tx, table) - if err != nil { - return 0, err - } - srcCols, err := queryStrings(ctx, src, d.columnsQuery, table) - if err != nil { - return 0, fmt.Errorf("list source columns: %w", err) - } - - names := make([]string, len(cols)) - quoted := make([]string, len(cols)) - var missing []string - for i, c := range cols { - names[i] = c.name - quoted[i] = d.quote(c.name) - if !slices.Contains(srcCols, c.name) { - missing = append(missing, c.name) - } - } - if len(missing) > 0 { - return 0, fmt.Errorf("source lacks columns %v; start the management server once with this release before migrating", missing) - } - - var hasRows bool - if err := tx.QueryRow(ctx, fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM %s)", pgx.Identifier{table}.Sanitize())).Scan(&hasRows); err != nil { - return 0, err - } - if hasRows { - return 0, fmt.Errorf("target already holds data") - } - - rows, err := src.QueryContext(ctx, fmt.Sprintf("SELECT %s FROM %s", strings.Join(quoted, ", "), d.quote(table))) - if err != nil { - return 0, fmt.Errorf("read source: %w", err) - } - defer rows.Close() - - return tx.CopyFrom(ctx, pgx.Identifier{table}, names, newRowSource(rows, cols)) -} - -func targetColumns(ctx context.Context, tx pgx.Tx, table string) ([]column, error) { - rows, err := tx.Query(ctx, `SELECT column_name, data_type FROM information_schema.columns - WHERE table_schema = current_schema() AND table_name = $1 ORDER BY ordinal_position`, table) - if err != nil { - return nil, err - } - return pgx.CollectRows(rows, func(row pgx.CollectableRow) (column, error) { - var c column - err := row.Scan(&c.name, &c.pgType) - return c, err - }) -} - -func parentsFirst(ctx context.Context, tx pgx.Tx, tables []string) ([]string, error) { - rows, err := tx.Query(ctx, `SELECT child.relname, parent.relname FROM pg_constraint c - JOIN pg_class child ON child.oid = c.conrelid - JOIN pg_class parent ON parent.oid = c.confrelid - JOIN pg_namespace n ON n.oid = child.relnamespace - WHERE c.contype = 'f' AND n.nspname = current_schema()`) - if err != nil { - return nil, fmt.Errorf("read foreign keys: %w", err) - } - parents := map[string][]string{} - var child, parent string - _, err = pgx.ForEachRow(rows, []any{&child, &parent}, func() error { - if child != parent { - parents[child] = append(parents[child], parent) - } - return nil - }) - if err != nil { - return nil, fmt.Errorf("read foreign keys: %w", err) - } - - pending := slices.Sorted(slices.Values(tables)) - done := make(map[string]bool, len(tables)) - ordered := make([]string, 0, len(tables)) - for len(pending) > 0 { - var next []string - for _, t := range pending { - if ready(parents[t], tables, done) { - ordered = append(ordered, t) - done[t] = true - } else { - next = append(next, t) - } - } - if len(next) == len(pending) { - return nil, fmt.Errorf("foreign keys form a cycle between %v", next) - } - pending = next - } - return ordered, nil -} - -func ready(parents, tables []string, done map[string]bool) bool { - for _, p := range parents { - if slices.Contains(tables, p) && !done[p] { - return false - } - } - return true -} - -// resetSequences moves serial sequences past the ids COPY inserted explicitly. -func resetSequences(ctx context.Context, tx pgx.Tx, tables []string) error { - rows, err := tx.Query(ctx, `SELECT table_name, column_name, pg_get_serial_sequence(quote_ident(table_name), column_name) - FROM information_schema.columns - WHERE table_schema = current_schema() AND table_name = ANY($1) - AND pg_get_serial_sequence(quote_ident(table_name), column_name) IS NOT NULL`, tables) - if err != nil { - return fmt.Errorf("list sequences: %w", err) - } - type sequence struct{ table, column, name string } - seqs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (sequence, error) { - var s sequence - err := row.Scan(&s.table, &s.column, &s.name) - return s, err - }) - if err != nil { - return fmt.Errorf("list sequences: %w", err) - } - - for _, s := range seqs { - q := fmt.Sprintf("SELECT setval('%s', COALESCE(MAX(%s), 0) + 1, false) FROM %s", - strings.ReplaceAll(s.name, "'", "''"), pgx.Identifier{s.column}.Sanitize(), pgx.Identifier{s.table}.Sanitize()) - if _, err := tx.Exec(ctx, q); err != nil { - return fmt.Errorf("reset sequence %s: %w", s.name, err) - } - } - return nil -} - -type rowSource struct { - rows *sql.Rows - cols []column - vals []any - ptrs []any -} - -func newRowSource(rows *sql.Rows, cols []column) *rowSource { - r := &rowSource{rows: rows, cols: cols, vals: make([]any, len(cols)), ptrs: make([]any, len(cols))} - for i := range r.vals { - r.ptrs[i] = &r.vals[i] - } - return r -} - -func (r *rowSource) Next() bool { return r.rows.Next() } - -func (r *rowSource) Err() error { return r.rows.Err() } - -func (r *rowSource) Values() ([]any, error) { - if err := r.rows.Scan(r.ptrs...); err != nil { - return nil, err - } - out := make([]any, len(r.vals)) - for i, v := range r.vals { - if v == nil { - continue - } - converted, err := convert(v, r.cols[i].pgType) - if err != nil { - return nil, fmt.Errorf("column %s: %w", r.cols[i].name, err) - } - out[i] = converted - } - return out, nil -} - -// convert adapts a non-NULL value to the target column type: MySQL and SQLite -// return text and numbers as bytes and booleans as integers. -func convert(v any, pgType string) (any, error) { - switch pgType { - case "boolean": - return toBool(v) - case "bytea": - return toBytes(v), nil - case "smallint", "integer", "bigint": - return toInt64(v) - case "real", "double precision": - return toFloat64(v) - case "numeric": - return toNumeric(v) - default: - return toText(v), nil - } -} - -func toBool(v any) (any, error) { - switch b := v.(type) { - case int64: - return b != 0, nil - case []byte: - return strconv.ParseBool(string(b)) - case string: - return strconv.ParseBool(b) - } - return v, nil -} - -func toBytes(v any) any { - if s, ok := v.(string); ok { - return []byte(s) - } - return v -} - -func toInt64(v any) (any, error) { - switch n := v.(type) { - case uint64: - if n > math.MaxInt64 { - return nil, fmt.Errorf("value %d overflows bigint", n) - } - return int64(n), nil - case []byte: - return strconv.ParseInt(string(n), 10, 64) - } - return v, nil -} - -func toFloat64(v any) (any, error) { - if b, ok := v.([]byte); ok { - return strconv.ParseFloat(string(b), 64) - } - return v, nil -} - -func toNumeric(v any) (any, error) { - var s string - switch x := v.(type) { - case []byte: - s = string(x) - case string: - s = x - default: - return v, nil - } - var n pgtype.Numeric - if err := n.Scan(s); err != nil { - return nil, err - } - return n, nil -} - -func toText(v any) any { - if b, ok := v.([]byte); ok { - return string(b) - } - return v -} - -func queryStrings(ctx context.Context, db *sql.DB, query string, args ...any) ([]string, error) { - rows, err := db.QueryContext(ctx, query, args...) - if err != nil { - return nil, err - } - defer rows.Close() - var out []string - for rows.Next() { - var s string - if err := rows.Scan(&s); err != nil { - return nil, err - } - out = append(out, s) - } - return out, rows.Err() -} - -func pgStrings(ctx context.Context, tx pgx.Tx, query string) ([]string, error) { - rows, err := tx.Query(ctx, query) - if err != nil { - return nil, err - } - return pgx.CollectRows(rows, pgx.RowTo[string]) -} diff --git a/tools/mysql-migrate/main.go b/tools/mysql-migrate/main.go deleted file mode 100644 index f9531c66a..000000000 --- a/tools/mysql-migrate/main.go +++ /dev/null @@ -1,242 +0,0 @@ -package main - -import ( - "context" - "database/sql" - "errors" - "flag" - "fmt" - "log/slog" - "net/url" - "os" - "strings" - "time" - - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgxpool" - log "github.com/sirupsen/logrus" - "gorm.io/driver/mysql" - "gorm.io/driver/postgres" - "gorm.io/driver/sqlite" - "gorm.io/gorm" - "gorm.io/gorm/logger" - - "github.com/netbirdio/netbird/idp/dex" - "github.com/netbirdio/netbird/management/internals/shared/db" - "github.com/netbirdio/netbird/management/server/activity" - "github.com/netbirdio/netbird/management/server/store" -) - -type config struct { - mysqlDSN string - postgresDSN string - eventsPostgresDSN string - authPostgresDSN string - eventsDB string - authDB string - mysqlTimezone string -} - -type storeMigration struct { - name string - source *sql.DB - dialect dialect - targetDSN string - schema func(ctx context.Context, dsn string) error - skipTables map[string]bool -} - -func main() { - cfg, err := parseFlags(os.Args[1:]) - if errors.Is(err, flag.ErrHelp) { - return - } - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(2) - } - - // Silence the store constructors' migration logs. - log.SetLevel(log.WarnLevel) - - if err := run(context.Background(), cfg); err != nil { - fmt.Fprintf(os.Stderr, "migration failed: %v\n", err) - os.Exit(1) - } -} - -func parseFlags(args []string) (*config, error) { - cfg := &config{} - fs := flag.NewFlagSet("netbird-mysql-migrate", flag.ContinueOnError) - fs.StringVar(&cfg.mysqlDSN, "mysql-dsn", "", "MySQL DSN of the management store (required)") - fs.StringVar(&cfg.postgresDSN, "postgres-dsn", "", "Postgres DSN to migrate all stores to (required)") - fs.StringVar(&cfg.eventsPostgresDSN, "events-postgres-dsn", "", "Postgres DSN for activity events (default --postgres-dsn)") - fs.StringVar(&cfg.authPostgresDSN, "auth-postgres-dsn", "", "Postgres DSN for the embedded IdP (default --postgres-dsn)") - fs.StringVar(&cfg.eventsDB, "events-db", "/var/lib/netbird/events.db", "SQLite file of the activity events, skipped when missing") - fs.StringVar(&cfg.authDB, "auth-db", "/var/lib/netbird/idp.db", "SQLite file of the embedded IdP, skipped when missing") - fs.StringVar(&cfg.mysqlTimezone, "mysql-timezone", "UTC", "time zone the management server ran in, e.g. Europe/Berlin") - if err := fs.Parse(args); err != nil { - return nil, err - } - if _, err := time.LoadLocation(cfg.mysqlTimezone); err != nil { - return nil, fmt.Errorf("--mysql-timezone: %w", err) - } - - if cfg.mysqlDSN == "" || cfg.postgresDSN == "" { - return nil, errors.New("--mysql-dsn and --postgres-dsn are required") - } - if cfg.eventsPostgresDSN == "" { - cfg.eventsPostgresDSN = cfg.postgresDSN - } - if cfg.authPostgresDSN == "" { - cfg.authPostgresDSN = cfg.postgresDSN - } - return cfg, nil -} - -func run(ctx context.Context, cfg *config) error { - migrations, closeSources, err := openSources(cfg) - if err != nil { - return err - } - defer closeSources() - - for _, m := range migrations { - if err := m.schema(ctx, m.targetDSN); err != nil { - return fmt.Errorf("create %s schema: %w", m.name, err) - } - } - - var ( - pools []*pgxpool.Pool - txs []pgx.Tx - ) - // Roll back first: pool Close waits for the connections open transactions hold. - defer func() { - for _, tx := range txs { - _ = tx.Rollback(ctx) - } - for _, pool := range pools { - pool.Close() - } - }() - - results := make([]string, 0, len(migrations)) - for _, m := range migrations { - pool, err := pgxpool.New(ctx, m.targetDSN) - if err != nil { - return fmt.Errorf("connect %s target: %w", m.name, err) - } - pools = append(pools, pool) - - tx, err := pool.Begin(ctx) - if err != nil { - return fmt.Errorf("begin %s transaction: %w", m.name, err) - } - txs = append(txs, tx) - - tables, rows, err := copyStore(ctx, m.source, m.dialect, tx, m.skipTables) - if err != nil { - return fmt.Errorf("%s: %w", m.name, err) - } - results = append(results, fmt.Sprintf("%s: %d rows from %d tables", m.name, rows, tables)) - } - - // Commit last so a failure in any store leaves every target empty. - for i, tx := range txs { - if err := tx.Commit(ctx); err != nil { - return fmt.Errorf("commit %s: %w", migrations[i].name, err) - } - } - - for _, r := range results { - fmt.Fprintln(os.Stdout, r) - } - fmt.Fprintln(os.Stdout, "Done. Point the store, activity and auth store settings at Postgres before starting the server.") - return nil -} - -func openSources(cfg *config) ([]storeMigration, func(), error) { - var opened []*gorm.DB - closeAll := func() { - for _, g := range opened { - if sqlDB, err := g.DB(); err == nil { - _ = sqlDB.Close() - } - } - } - - open := func(dialector gorm.Dialector) (*sql.DB, error) { - g, err := gorm.Open(dialector, &gorm.Config{Logger: logger.Discard}) - if err != nil { - return nil, err - } - opened = append(opened, g) - return g.DB() - } - - mysqlDB, err := open(mysql.Open(mysqlDSN(cfg.mysqlDSN, cfg.mysqlTimezone))) - if err != nil { - closeAll() - return nil, nil, fmt.Errorf("open MySQL: %w", err) - } - migrations := []storeMigration{{ - name: "management", source: mysqlDB, dialect: mysqlDialect, - targetDSN: cfg.postgresDSN, schema: createManagementSchema, - }} - - sqliteStores := []storeMigration{ - {name: "activity", targetDSN: cfg.eventsPostgresDSN, schema: createActivitySchema}, - {name: "auth", targetDSN: cfg.authPostgresDSN, schema: createAuthSchema, skipTables: map[string]bool{"migrations": true}}, - } - for i, file := range []string{cfg.eventsDB, cfg.authDB} { - m := sqliteStores[i] - if _, err := os.Stat(file); errors.Is(err, os.ErrNotExist) { - fmt.Fprintf(os.Stdout, "%s: %s not found, skipping\n", m.name, file) - continue - } - m.source, err = open(sqlite.Open("file:" + file + "?mode=ro")) - if err != nil { - closeAll() - return nil, nil, fmt.Errorf("open %s: %w", file, err) - } - m.dialect = sqliteDialect - migrations = append(migrations, m) - } - - return migrations, closeAll, nil -} - -func mysqlDSN(dsn, timezone string) string { - return strings.TrimSuffix(db.MysqlDSN(dsn), "&loc=Local") + "&loc=" + url.QueryEscape(timezone) -} - -func createManagementSchema(ctx context.Context, dsn string) error { - s, err := store.NewPostgresqlStore(ctx, dsn, nil, false) - if err != nil { - return err - } - return s.Close(ctx) -} - -func createActivitySchema(_ context.Context, dsn string) error { - g, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: logger.Discard}) - if err != nil { - return err - } - defer func() { - if sqlDB, err := g.DB(); err == nil { - _ = sqlDB.Close() - } - }() - return g.AutoMigrate(&activity.Event{}, &activity.DeletedUser{}) -} - -func createAuthSchema(_ context.Context, dsn string) error { - cfg := dex.Storage{Type: "postgres", Config: map[string]any{"dsn": dsn}} - s, err := cfg.OpenStorage(slog.New(slog.DiscardHandler)) - if err != nil { - return err - } - return s.Close() -} diff --git a/tools/mysql-migrate/main_test.go b/tools/mysql-migrate/main_test.go deleted file mode 100644 index 59d444440..000000000 --- a/tools/mysql-migrate/main_test.go +++ /dev/null @@ -1,225 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "log/slog" - "os" - "path/filepath" - "runtime" - "strings" - "testing" - "time" - - dexstorage "github.com/dexidp/dex/storage" - "github.com/jackc/pgx/v5/pgxpool" - "github.com/rs/xid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "gorm.io/driver/mysql" - "gorm.io/gorm" - - "github.com/netbirdio/netbird/idp/dex" - "github.com/netbirdio/netbird/management/internals/shared/db" - "github.com/netbirdio/netbird/management/server/activity" - activitystore "github.com/netbirdio/netbird/management/server/activity/store" - "github.com/netbirdio/netbird/management/server/store" - "github.com/netbirdio/netbird/management/server/testutil" - "github.com/netbirdio/netbird/management/server/types" - "github.com/netbirdio/netbird/util/crypt" -) - -const testAccountID = "bf1c8084-ba50-4ce7-9439-34653001fc3b" - -func TestRun(t *testing.T) { - if runtime.GOOS == "windows" || (os.Getenv("CI") == "true" && runtime.GOOS != "linux") { - t.Skip("needs Docker with Linux containers for MySQL and Postgres") - } - // Seed in UTC to match the --mysql-timezone default. - time.Local = time.UTC - ctx := context.Background() - dir := t.TempDir() - key, err := crypt.GenerateKey() - require.NoError(t, err) - - mysqlDSN := newMySQLDatabase(t, "run_src") - pgDSN := newPostgresDatabase(t, ctx, "run_dst") - mysqlStore := seedMySQL(t, ctx, mysqlDSN) - eventID := seedEvents(t, ctx, dir, key) - seedIdP(t, ctx, filepath.Join(dir, "idp.db")) - - cfg, err := parseFlags([]string{ - "--mysql-dsn", mysqlDSN, - "--postgres-dsn", pgDSN, - "--events-db", filepath.Join(dir, "events.db"), - "--auth-db", filepath.Join(dir, "idp.db"), - }) - require.NoError(t, err) - require.NoError(t, run(ctx, cfg)) - - t.Run("management", func(t *testing.T) { - pgStore, err := store.NewPostgresqlStore(ctx, pgDSN, nil, true) - require.NoError(t, err) - t.Cleanup(func() { _ = pgStore.Close(ctx) }) - - assert.JSONEq(t, accountJSON(t, ctx, mysqlStore), accountJSON(t, ctx, pgStore), - "the account read from Postgres must match the one in MySQL") - }) - - t.Run("activity", func(t *testing.T) { - t.Setenv("NB_ACTIVITY_EVENT_STORE_ENGINE", "postgres") - t.Setenv("NB_ACTIVITY_EVENT_POSTGRES_DSN", pgDSN) - events, err := activitystore.NewSqlStore(ctx, dir, key) - require.NoError(t, err) - t.Cleanup(func() { _ = events.Close(ctx) }) - - got, err := events.Get(ctx, testAccountID, 0, 10, false) - require.NoError(t, err) - require.Len(t, got, 1, "the migrated event must be readable") - assert.Equal(t, eventID, got[0].ID, "the event must keep its id") - - next, err := events.Save(ctx, &activity.Event{Timestamp: time.Now(), AccountID: testAccountID, InitiatorID: "u", TargetID: "p"}) - require.NoError(t, err) - assert.Greater(t, next.ID, eventID, "new events must continue after the migrated ids") - }) - - t.Run("auth", func(t *testing.T) { - idp := dex.Storage{Type: "postgres", Config: map[string]any{"dsn": pgDSN}} - s, err := idp.OpenStorage(slog.New(slog.DiscardHandler)) - require.NoError(t, err) - t.Cleanup(func() { _ = s.Close() }) - - pw, err := s.GetPassword(ctx, "admin@example.com") - require.NoError(t, err) - assert.Equal(t, []byte("hash"), pw.Hash, "the IdP password must survive the migration") - }) - - t.Run("refuses a target that holds data", func(t *testing.T) { - assert.ErrorContains(t, run(ctx, cfg), "target already holds data") - }) -} - -func seedMySQL(t *testing.T, ctx context.Context, dsn string) *store.SqlStore { - t.Helper() - t.Setenv("NETBIRD_STORE_ENGINE", string(types.SqliteStoreEngine)) - seed, cleanup, err := store.NewTestStoreFromSQL(ctx, "../../management/server/testdata/extended-store.sql", t.TempDir()) - require.NoError(t, err) - t.Cleanup(cleanup) - - mysqlStore, err := store.NewMysqlStoreFromSqlStore(ctx, seed.(*store.SqlStore), dsn, nil) - require.NoError(t, err) - t.Cleanup(func() { _ = mysqlStore.Close(ctx) }) - return mysqlStore -} - -func seedEvents(t *testing.T, ctx context.Context, dir, key string) uint64 { - t.Helper() - events, err := activitystore.NewSqlStore(ctx, dir, key) - require.NoError(t, err) - defer events.Close(ctx) - - event, err := events.Save(ctx, &activity.Event{ - Timestamp: time.Now(), Activity: activity.PeerAddedByUser, - AccountID: testAccountID, InitiatorID: "user", TargetID: "peer", - Meta: map[string]any{"name": "peer"}, - }) - require.NoError(t, err) - return event.ID -} - -func seedIdP(t *testing.T, ctx context.Context, file string) { - t.Helper() - idp := dex.Storage{Type: "sqlite3", Config: map[string]any{"file": file}} - s, err := idp.OpenStorage(slog.New(slog.DiscardHandler)) - require.NoError(t, err) - defer s.Close() - - require.NoError(t, s.CreatePassword(ctx, dexstorage.Password{ - Email: "admin@example.com", Hash: []byte("hash"), Username: "admin", UserID: "admin-id", - })) -} - -func accountJSON(t *testing.T, ctx context.Context, s store.Store) string { - t.Helper() - var account *types.Account - require.NoError(t, s.ExecuteInTransaction(ctx, func(tx store.Store) error { - var err error - account, err = tx.GetAccount(ctx, testAccountID) - return err - })) - out, err := json.Marshal(account) - require.NoError(t, err) - return string(out) -} - -func TestMySQLTimezone(t *testing.T) { - if runtime.GOOS == "windows" || (os.Getenv("CI") == "true" && runtime.GOOS != "linux") { - t.Skip("needs Docker with Linux containers for MySQL and Postgres") - } - ctx := context.Background() - mysqlDSN := newMySQLDatabase(t, "tz_src") - pgDSN := newPostgresDatabase(t, ctx, "tz_dst") - ms := seedMySQL(t, ctx, mysqlDSN) - require.NoError(t, ms.GetDB().Exec("UPDATE accounts SET created_at = '2026-01-02 03:04:05' WHERE id = ?", testAccountID).Error) - - cfg, err := parseFlags([]string{ - "--mysql-dsn", mysqlDSN, - "--postgres-dsn", pgDSN, - "--mysql-timezone", "Asia/Tokyo", - "--events-db", filepath.Join(t.TempDir(), "none.db"), - "--auth-db", filepath.Join(t.TempDir(), "none.db"), - }) - require.NoError(t, err) - require.NoError(t, run(ctx, cfg)) - - pool, err := pgxpool.New(ctx, pgDSN) - require.NoError(t, err) - t.Cleanup(pool.Close) - var got time.Time - require.NoError(t, pool.QueryRow(ctx, "SELECT created_at FROM accounts WHERE id = $1", testAccountID).Scan(&got)) - want := time.Date(2026, 1, 1, 18, 4, 5, 0, time.UTC) - assert.True(t, want.Equal(got), "created_at must keep the instant the server wrote, got %s", got.UTC()) -} - -func TestParseFlagsRejectsUnknownTimezone(t *testing.T) { - _, err := parseFlags([]string{"--mysql-dsn", "m", "--postgres-dsn", "p", "--mysql-timezone", "Mars/Base"}) - assert.ErrorContains(t, err, "--mysql-timezone") -} - -// newMySQLDatabase creates a database unique to this run in the shared -// container, so repeated runs (-count) don't collide, and drops it afterwards. -func newMySQLDatabase(t *testing.T, prefix string) string { - t.Helper() - _, base, err := testutil.CreateMysqlTestContainer() - require.NoError(t, err) - admin, err := gorm.Open(mysql.Open(db.MysqlDSN(base)), &gorm.Config{}) - require.NoError(t, err) - name := prefix + "_" + xid.New().String() - require.NoError(t, admin.Exec("CREATE DATABASE "+name).Error) - t.Cleanup(func() { - _ = admin.Exec("DROP DATABASE IF EXISTS " + name).Error - if sqlDB, err := admin.DB(); err == nil { - _ = sqlDB.Close() - } - }) - return base[:strings.LastIndex(base, "/")] + "/" + name -} - -// newPostgresDatabase is the Postgres counterpart of newMySQLDatabase. -func newPostgresDatabase(t *testing.T, ctx context.Context, prefix string) string { - t.Helper() - _, base, err := testutil.CreatePostgresTestContainer() - require.NoError(t, err) - // Dex connects through lib/pq, which requires TLS unless told otherwise. - base = strings.TrimSuffix(base, "?") - admin, err := pgxpool.New(ctx, base+"?sslmode=disable") - require.NoError(t, err) - name := prefix + "_" + xid.New().String() - _, err = admin.Exec(ctx, "CREATE DATABASE "+name) - require.NoError(t, err) - t.Cleanup(func() { - _, _ = admin.Exec(context.Background(), "DROP DATABASE IF EXISTS "+name+" WITH (FORCE)") - admin.Close() - }) - return base[:strings.LastIndex(base, "/")] + "/" + name + "?sslmode=disable" -}