sendnrw/netbird
2
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2026-04-19 16:56:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

MySQL Support (#2837)

Browse Source 
* Update store.go

* Update sql_store.go

* Update store.go

* Update golang-test-linux.yml

* Update store.go

* Update go.mod

* Update go.mod

* Update go.sum

* Update store.go

* Update sql_store.go

* TestContainer

* Update go.sum

* Update store.go

* TestUtil Duplicate

* dsn fix

* go mod tidy

* NETBIRD_STORE_ENGINE_MYSQL_DSN

* Skip Test

* Update test-infrastructure-files.yml

* Update test-infrastructure-files.yml

* MYSQL_ROOT_PASSWORD added

* Update test-infrastructure-files.yml

* Update store.go

* Debug + Mysql JSON Query

* swicth/case convert

* Update store.go

* Update store.go

* Debug

* MySQL Test Version Change

* Root Test

* Ignore other sql tests.

* MySQL Connection Fix

* enable other tests

* The word "key" is a reserved word in MySQL.

* Remove Debugs

* Update sql_store.go

* Added default null value for datetime.

* Added default null value for datetime.

* MySQL Hooks

* MySQL Config File

* remove default values

* test timeout change

* MySQL max lifetime change

* WithConfigFile

* disable other tests

* Update mysql.cnf

* Update golang-test-linux.yml

* Delete sql_hooks.go

* enable other tests

* test timeout change

* update packets

* Fix the Inactivity Expiration problem

* Update sql_store.go

* Update mysql.cnf

* Update sql_store.go

* Update sql_store.go

* timeout change

* MySQL Connection LifeTime Change

* TestContainers have been optimized.

* Update store_ios.go

* Update sql_store.go

* timeout fix

* fix migration (setup keys)

* Update event.go

* Add disable option for event activities.

* Revert "Update event.go"

* Update event.go

* Fix Gorm Mysql Bug

* update go-jose module

* containerd module update

* containerd downgrade

* Revert commits

* Revert "Revert commits"

This reverts commit 62b3eac799.

* Revert "containerd downgrade"

This reverts commit 4e46108915.

* Revert "containerd module update"

This reverts commit e8cfa87d16.

* Revert "update go-jose module"

This reverts commit 1fabdc7606.
This commit is contained in:
İsmail
2024-12-23 13:06:13 +03:00
committed by GitHub
parent 37ad370344
commit 215c9047ba
16 changed files with 366 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
management/server/sql_store.go
View File

@@ -16,6 +16,7 @@ import (
"time"
log "github.com/sirupsen/logrus"
"gorm.io/driver/mysql"
"gorm.io/driver/postgres"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
@@ -36,6 +37,7 @@ const (
storeSqliteFileName = "store.db"
idQueryCondition = "id = ?"
keyQueryCondition = "key = ?"
mysqlKeyQueryCondition = "`key` = ?"
accountAndIDQueryCondition = "account_id = ? and id = ?"
accountAndIDsQueryCondition = "account_id = ? AND id IN ?"
accountIDCondition = "account_id = ?"
@@ -80,6 +82,12 @@ func NewSqlStore(ctx context.Context, db *gorm.DB, storeEngine StoreEngine, metr
sql.SetMaxOpenConns(conns)
if storeEngine == MysqlStoreEngine {
sql.SetConnMaxLifetime(time.Minute * 2)
sql.SetConnMaxIdleTime(time.Minute * 2)
sql.SetMaxIdleConns(conns)
}
log.WithContext(ctx).Infof("Set max open db connections to %d", conns)
if err := migrate(ctx, db); err != nil {
@@ -97,6 +105,15 @@ func NewSqlStore(ctx context.Context, db *gorm.DB, storeEngine StoreEngine, metr
return &SqlStore{db: db, storeEngine: storeEngine, metrics: metrics, installationPK: 1}, nil
}
func GetKeyQueryCondition(s *SqlStore) string {
if s.storeEngine == MysqlStoreEngine {
return mysqlKeyQueryCondition
}
return keyQueryCondition
}
// AcquireGlobalLock acquires global lock across all the accounts and returns a function that releases the lock
func (s *SqlStore) AcquireGlobalLock(ctx context.Context) (unlock func()) {
log.WithContext(ctx).Tracef("acquiring global lock")
@@ -393,7 +410,7 @@ func (s *SqlStore) SavePeerLocation(accountID string, peerWithLocation *nbpeer.P
return result.Error
}
if result.RowsAffected == 0 {
if result.RowsAffected == 0 && s.storeEngine != MysqlStoreEngine {
return status.Errorf(status.NotFound, peerNotFoundFMT, peerWithLocation.ID)
}
@@ -483,7 +500,8 @@ func (s *SqlStore) GetAccountIDByPrivateDomain(ctx context.Context, lockStrength
func (s *SqlStore) GetAccountBySetupKey(ctx context.Context, setupKey string) (*Account, error) {
var key SetupKey
result := s.db.Select("account_id").First(&key, keyQueryCondition, setupKey)
result := s.db.Select("account_id").First(&key, GetKeyQueryCondition(s), setupKey)
if result.Error != nil {
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
return nil, status.NewSetupKeyNotFoundError(setupKey)
@@ -711,7 +729,8 @@ func (s *SqlStore) GetAccountByPeerID(ctx context.Context, peerID string) (*Acco
func (s *SqlStore) GetAccountByPeerPubKey(ctx context.Context, peerKey string) (*Account, error) {
var peer nbpeer.Peer
result := s.db.Select("account_id").First(&peer, keyQueryCondition, peerKey)
result := s.db.Select("account_id").First(&peer, GetKeyQueryCondition(s), peerKey)
if result.Error != nil {
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
return nil, status.Errorf(status.NotFound, "account not found: index lookup failed")
@@ -729,7 +748,7 @@ func (s *SqlStore) GetAccountByPeerPubKey(ctx context.Context, peerKey string) (
func (s *SqlStore) GetAccountIDByPeerPubKey(ctx context.Context, peerKey string) (string, error) {
var peer nbpeer.Peer
var accountID string
result := s.db.Model(&peer).Select("account_id").Where(keyQueryCondition, peerKey).First(&accountID)
result := s.db.Model(&peer).Select("account_id").Where(GetKeyQueryCondition(s), peerKey).First(&accountID)
if result.Error != nil {
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
return "", status.Errorf(status.NotFound, "account not found: index lookup failed")
@@ -755,7 +774,7 @@ func (s *SqlStore) GetAccountIDByUserID(userID string) (string, error) {
func (s *SqlStore) GetAccountIDBySetupKey(ctx context.Context, setupKey string) (string, error) {
var accountID string
result := s.db.Model(&SetupKey{}).Select("account_id").Where(keyQueryCondition, setupKey).First(&accountID)
result := s.db.Model(&SetupKey{}).Select("account_id").Where(GetKeyQueryCondition(s), setupKey).First(&accountID)
if result.Error != nil {
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
return "", status.NewSetupKeyNotFoundError(setupKey)
@@ -828,7 +847,8 @@ func (s *SqlStore) GetAccountNetwork(ctx context.Context, lockStrength LockingSt
func (s *SqlStore) GetPeerByPeerPubKey(ctx context.Context, lockStrength LockingStrength, peerKey string) (*nbpeer.Peer, error) {
var peer nbpeer.Peer
result := s.db.Clauses(clause.Locking{Strength: string(lockStrength)}).First(&peer, keyQueryCondition, peerKey)
result := s.db.Clauses(clause.Locking{Strength: string(lockStrength)}).First(&peer, GetKeyQueryCondition(s), peerKey)
if result.Error != nil {
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
return nil, status.Errorf(status.NotFound, "peer not found")
@@ -921,11 +941,22 @@ func NewPostgresqlStore(ctx context.Context, dsn string, metrics telemetry.AppMe
return NewSqlStore(ctx, db, PostgresStoreEngine, metrics)
}
// NewMysqlStore creates a new MySQL store.
func NewMysqlStore(ctx context.Context, dsn string, metrics telemetry.AppMetrics) (*SqlStore, error) {
db, err := gorm.Open(mysql.Open(dsn+"?charset=utf8&parseTime=True&loc=Local"), getGormConfig())
if err != nil {
return nil, err
}
return NewSqlStore(ctx, db, MysqlStoreEngine, metrics)
}
func getGormConfig() *gorm.Config {
return &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent),
CreateBatchSize: 400,
PrepareStmt: true,
Logger: logger.Default.LogMode(logger.Silent),
CreateBatchSize: 400,
PrepareStmt: true,
SkipDefaultTransaction: true,
}
}
@@ -938,6 +969,15 @@ func newPostgresStore(ctx context.Context, metrics telemetry.AppMetrics) (Store,
return NewPostgresqlStore(ctx, dsn, metrics)
}
// newMysqlStore initializes a new MySQL store.
func newMysqlStore(ctx context.Context, metrics telemetry.AppMetrics) (Store, error) {
dsn, ok := os.LookupEnv(mysqlDsnEnv)
if !ok {
return nil, fmt.Errorf("%s is not set", mysqlDsnEnv)
}
return NewMysqlStore(ctx, dsn, metrics)
}
// NewSqliteStoreFromFileStore restores a store from FileStore and stores SQLite DB in the file located in datadir.
func NewSqliteStoreFromFileStore(ctx context.Context, fileStore *FileStore, dataDir string, metrics telemetry.AppMetrics) (*SqlStore, error) {
store, err := NewSqliteStore(ctx, dataDir, metrics)
@@ -982,10 +1022,33 @@ func NewPostgresqlStoreFromSqlStore(ctx context.Context, sqliteStore *SqlStore,
return store, nil
}
// NewMysqlStoreFromSqlStore restores a store from SqlStore and stores MySQL DB.
func NewMysqlStoreFromSqlStore(ctx context.Context, sqliteStore *SqlStore, dsn string, metrics telemetry.AppMetrics) (*SqlStore, error) {
store, err := NewMysqlStore(ctx, dsn, metrics)
if err != nil {
return nil, err
}
err = store.SaveInstallationID(ctx, sqliteStore.GetInstallationID())
if err != nil {
return nil, err
}
for _, account := range sqliteStore.GetAllAccounts(ctx) {
err := store.SaveAccount(ctx, account)
if err != nil {
return nil, err
}
}
return store, nil
}
func (s *SqlStore) GetSetupKeyBySecret(ctx context.Context, lockStrength LockingStrength, key string) (*SetupKey, error) {
var setupKey SetupKey
result := s.db.Clauses(clause.Locking{Strength: string(lockStrength)}).
First(&setupKey, keyQueryCondition, key)
First(&setupKey, GetKeyQueryCondition(s), key)
if result.Error != nil {
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
return nil, status.NewSetupKeyNotFoundError(key)
@@ -1222,9 +1285,13 @@ func (s *SqlStore) GetGroupByName(ctx context.Context, lockStrength LockingStren
// TODO: This fix is accepted for now, but if we need to handle this more frequently
// we may need to reconsider changing the types.
query := s.db.Clauses(clause.Locking{Strength: string(lockStrength)}).Preload(clause.Associations)
if s.storeEngine == PostgresStoreEngine {
switch s.storeEngine {
case PostgresStoreEngine:
query = query.Order("json_array_length(peers::json) DESC")
} else {
case MysqlStoreEngine:
query = query.Order("JSON_LENGTH(JSON_EXTRACT(peers, \"$\")) DESC")
default:
query = query.Order("json_array_length(peers) DESC")
}