From 158936fb15596690003d602c5df918f6522b97c1 Mon Sep 17 00:00:00 2001 From: pascal-fischer <32096965+pascal-fischer@users.noreply.github.com> Date: Thu, 3 Oct 2024 15:50:35 +0200 Subject: [PATCH 01/12] [management] Remove file store (#2689) --- client/cmd/testutil_test.go | 13 +- client/internal/engine_test.go | 21 +- client/server/server_test.go | 2 +- client/testdata/store.json | 38 - client/testdata/store.sqlite | Bin 0 -> 163840 bytes management/client/client_test.go | 28 +- management/server/account_test.go | 2 +- management/server/dns_test.go | 2 +- management/server/file_store.go | 791 +----------------- management/server/file_store_test.go | 655 --------------- management/server/management_proto_test.go | 49 +- management/server/management_test.go | 9 +- management/server/nameserver_test.go | 2 +- management/server/peer_test.go | 18 +- management/server/route_test.go | 4 +- management/server/sql_store.go | 22 + management/server/sql_store_test.go | 144 ++-- management/server/store.go | 56 +- management/server/store_test.go | 21 +- .../server/testdata/extended-store.json | 120 --- .../server/testdata/extended-store.sqlite | Bin 0 -> 163840 bytes management/server/testdata/store.json | 88 -- management/server/testdata/store.sqlite | Bin 0 -> 163840 bytes .../server/testdata/store_policy_migrate.json | 116 --- .../testdata/store_policy_migrate.sqlite | Bin 0 -> 163840 bytes .../testdata/store_with_expired_peers.json | 130 --- .../testdata/store_with_expired_peers.sqlite | Bin 0 -> 163840 bytes management/server/testdata/storev1.json | 154 ---- management/server/testdata/storev1.sqlite | Bin 0 -> 163840 bytes management/server/user_test.go | 65 +- 30 files changed, 259 insertions(+), 2291 deletions(-) delete mode 100644 client/testdata/store.json create mode 100644 client/testdata/store.sqlite delete mode 100644 management/server/file_store_test.go delete mode 100644 management/server/testdata/extended-store.json create mode 100644 management/server/testdata/extended-store.sqlite delete mode 100644 management/server/testdata/store.json create mode 100644 management/server/testdata/store.sqlite delete mode 100644 management/server/testdata/store_policy_migrate.json create mode 100644 management/server/testdata/store_policy_migrate.sqlite delete mode 100644 management/server/testdata/store_with_expired_peers.json create mode 100644 management/server/testdata/store_with_expired_peers.sqlite delete mode 100644 management/server/testdata/storev1.json create mode 100644 management/server/testdata/storev1.sqlite diff --git a/client/cmd/testutil_test.go b/client/cmd/testutil_test.go index f0dc8bf21..033d1bb6a 100644 --- a/client/cmd/testutil_test.go +++ b/client/cmd/testutil_test.go @@ -3,7 +3,6 @@ package cmd import ( "context" "net" - "path/filepath" "testing" "time" @@ -34,18 +33,12 @@ func startTestingServices(t *testing.T) string { if err != nil { t.Fatal(err) } - testDir := t.TempDir() - config.Datadir = testDir - err = util.CopyFileContents("../testdata/store.json", filepath.Join(testDir, "store.json")) - if err != nil { - t.Fatal(err) - } _, signalLis := startSignal(t) signalAddr := signalLis.Addr().String() config.Signal.URI = signalAddr - _, mgmLis := startManagement(t, config) + _, mgmLis := startManagement(t, config, "../testdata/store.sqlite") mgmAddr := mgmLis.Addr().String() return mgmAddr } @@ -70,7 +63,7 @@ func startSignal(t *testing.T) (*grpc.Server, net.Listener) { return s, lis } -func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Listener) { +func startManagement(t *testing.T, config *mgmt.Config, testFile string) (*grpc.Server, net.Listener) { t.Helper() lis, err := net.Listen("tcp", ":0") @@ -78,7 +71,7 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste t.Fatal(err) } s := grpc.NewServer() - store, cleanUp, err := mgmt.NewTestStoreFromJson(context.Background(), config.Datadir) + store, cleanUp, err := mgmt.NewTestStoreFromSqlite(context.Background(), testFile, t.TempDir()) if err != nil { t.Fatal(err) } diff --git a/client/internal/engine_test.go b/client/internal/engine_test.go index 29a8439a2..3d1983c6b 100644 --- a/client/internal/engine_test.go +++ b/client/internal/engine_test.go @@ -6,7 +6,6 @@ import ( "net" "net/netip" "os" - "path/filepath" "runtime" "strings" "sync" @@ -824,20 +823,6 @@ func TestEngine_UpdateNetworkMapWithDNSUpdate(t *testing.T) { func TestEngine_MultiplePeers(t *testing.T) { // log.SetLevel(log.DebugLevel) - dir := t.TempDir() - - err := util.CopyFileContents("../testdata/store.json", filepath.Join(dir, "store.json")) - if err != nil { - t.Fatal(err) - } - defer func() { - err = os.Remove(filepath.Join(dir, "store.json")) //nolint - if err != nil { - t.Fatal(err) - return - } - }() - ctx, cancel := context.WithCancel(CtxInitState(context.Background())) defer cancel() @@ -847,7 +832,7 @@ func TestEngine_MultiplePeers(t *testing.T) { return } defer sigServer.Stop() - mgmtServer, mgmtAddr, err := startManagement(t, dir) + mgmtServer, mgmtAddr, err := startManagement(t, t.TempDir(), "../testdata/store.sqlite") if err != nil { t.Fatal(err) return @@ -1070,7 +1055,7 @@ func startSignal(t *testing.T) (*grpc.Server, string, error) { return s, lis.Addr().String(), nil } -func startManagement(t *testing.T, dataDir string) (*grpc.Server, string, error) { +func startManagement(t *testing.T, dataDir, testFile string) (*grpc.Server, string, error) { t.Helper() config := &server.Config{ @@ -1095,7 +1080,7 @@ func startManagement(t *testing.T, dataDir string) (*grpc.Server, string, error) } s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp)) - store, cleanUp, err := server.NewTestStoreFromJson(context.Background(), config.Datadir) + store, cleanUp, err := server.NewTestStoreFromSqlite(context.Background(), testFile, config.Datadir) if err != nil { return nil, "", err } diff --git a/client/server/server_test.go b/client/server/server_test.go index 9b18df4d3..e534ad7e2 100644 --- a/client/server/server_test.go +++ b/client/server/server_test.go @@ -110,7 +110,7 @@ func startManagement(t *testing.T, signalAddr string, counter *int) (*grpc.Serve return nil, "", err } s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp)) - store, cleanUp, err := server.NewTestStoreFromJson(context.Background(), config.Datadir) + store, cleanUp, err := server.NewTestStoreFromSqlite(context.Background(), "", config.Datadir) if err != nil { return nil, "", err } diff --git a/client/testdata/store.json b/client/testdata/store.json deleted file mode 100644 index 8236f2703..000000000 --- a/client/testdata/store.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "Accounts": { - "bf1c8084-ba50-4ce7-9439-34653001fc3b": { - "Id": "bf1c8084-ba50-4ce7-9439-34653001fc3b", - "SetupKeys": { - "A2C8E62B-38F5-4553-B31E-DD66C696CEBB": { - "Key": "A2C8E62B-38F5-4553-B31E-DD66C696CEBB", - "Name": "Default key", - "Type": "reusable", - "CreatedAt": "2021-08-19T20:46:20.005936822+02:00", - "ExpiresAt": "2321-09-18T20:46:20.005936822+02:00", - "Revoked": false, - "UsedTimes": 0 - - } - }, - "Network": { - "Id": "af1c8024-ha40-4ce2-9418-34653101fc3c", - "Net": { - "IP": "100.64.0.0", - "Mask": "//8AAA==" - }, - "Dns": null - }, - "Peers": {}, - "Users": { - "edafee4e-63fb-11ec-90d6-0242ac120003": { - "Id": "edafee4e-63fb-11ec-90d6-0242ac120003", - "Role": "admin" - }, - "f4f6d672-63fb-11ec-90d6-0242ac120003": { - "Id": "f4f6d672-63fb-11ec-90d6-0242ac120003", - "Role": "user" - } - } - } - } -} \ No newline at end of file diff --git a/client/testdata/store.sqlite b/client/testdata/store.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..118c2bebc9f1fd29751627c36304d301ba156781 GIT binary patch literal 163840 zcmeI5Piz}ke#beIMbfe*#>plVXQN$8iDR>oY?6`{$qU1kWjbrbmJ`Xwu3-ej5&1|> zM9wfXL)k*G1!yP1Vu512XpTMf)I$&3Vu3d2JrzCdvA|x7_S#zy1+s_!=FPm}d*slP z>)j$DzO>BYy!U(mKEL1l{ob45pPi3xTRzJ-9Jg)y`Q_A0DNRfLc|M;?rQW3f7wEru zyg@Hch!Z+$((6$#-%MR}|8^=6&V3Weyqf!F`ZuQ^PG3n+P3NY*p4yq*o4PQ0LNh16 zPW?9Z75$^fyDz8pYiG5TW!Kqb@6#4_&@j8c!_SST>vTJwQ3{W*^yXg5=pU_Xy{kOi zsy{Y5%=H}GY#C;)#yrpPoqc9|M%QDmVbzm!&ung2HttpOx3+FnewwcyT}?HAcPn2_ zuB>{Y8Z}leJlUDe=ZVVk!Lx6**w55nFR7B}y1?cCemxV3dJUv2DjU3f;vGC@2iBa5}AD)khf|*P2n{hnd~!L38Rb)tvbGsbO;(loLLN zN)MRR?UQN!yslZX#fC-q8Nngx=}~o)fLdvgQ22Q!!;SNo?Z`(_6}+CTYMeZ+dW3QE zCPHuIF~8d}qy(!Y4699MY3w3FY&wSHDPsG~wOLC^syVJOf+{c7X_KhyFL75&CJP3G_n#T`C}o8vQdIHDfk&1IkE=EB-3jM6MRd63TN*XFg} z+1Llg(_Ido{lv_$Q|f0*!8UYI{j34k_w=0;8GU|U`%>r5sxQ}nHKn?(e#Ue7E_tFz zA!kIS3(cIyt*ni@onT2+A4qlC0WC;nvn5lfg&8fo$YaDYFlk{`O$gCl&ulWIWwkA8 zJ1(7Iy3b)oePSSH$f008+;rB%ndHBJkF#brE5<~$~fN$AN z>W1ZUpK4H)A+ri%$-s&*7UjPF7!LQ@1*DF|f-Y-zTc#Ur5)Lf($QWLS#?sM#km5(j zY>x8!vGgYt?V}RIu^~yS)-tWO;;a=Zv(<7QjY34rUe~1UC%)aB3#DT5;_Ax9;>F^^;=&!%+o!Zkm)15mHm+V> z_}Rmcx4!z--BfCtI;Q1*pQ1lJKmY_l00ck)1V8`;KmY_l00ck)1ioVg&QIPyd7KLl z;r{;*soWpFV*^BYK>!3m00ck)1V8`;KmY_l00ck)1dc^ua&qQm`2PP~DwjJ}5eR_* z2!H?xfB*=900@8p2!H?xfWSBq=-FCs=KlS+=5}^=ve|2Avf1O@MBgZHu2ojc*O!*p zZeCtmxqNwf>H2c1vUKCd>gwj|`s!xo`t=*EVRl=7p1uvqW%QjL^u-hOX8+Psac!xz zo-Y@#tgK!s)0_2+m)Dn9*UIJa{r=_UKxBQXv^HAg=vs>O>4GKxmoFCSQv~$)E`NUb zG4B75gV|yXAOHd&00JNY0w4eaAOHd&00JN|nm};>AD{mp%?CLk00JNY0w4eaAOHd& z00JNY0wC~hCve=){|E2?ryu>p0|Y<-1V8`;KmY_l00ck)1V8`;Kwyjs1o!{3{~u!o zV`v}%0w4eaAOHd&00JNY0w4eaAV37L|A!fX00@8p2!H?xfB*=900@8p2!O!&6Ttp| z{B?{Gf&d7B00@8p2!H?xfB*=900@9U@cI9V*?&#t>hy#M2!H?xfB*=900@8p2!H?x zfB*=5+X?hupP0RW|J7`E=1lh5oma9mjg`h~ef7Qa(&}0Qiavk<2!H?xfB*=900@8p2!H?xfPg|E*#95r zO$&Vge;h&l{-44ZCm;X#3c|znss$B&$jB1jSh1?$2MDrS*tP6Gkj;C z*`5*n$f9A@L&SgDHj;aAW^=o;aj%lUwRNNN(|q;lu2%DRxAN8G%BmNt(SWLjCp)tl zeSKZ~e8V?)TP(Ria%wK>)4h%Bw=2A@KWb$OdZ^|v(9o-SpFQ>$<1s|1w%KMPYtQud zSUuF9NMnyXmTUT!V;gnTC(+>9x7uv6T5}n_t{c0DLRh4jzAV==JwLDo>Ox%j!|hvl zHntz+Kd3widskT8yt}n?Z+qj`*1deSvCoaiGdiZ>dq%UGzj=4Na_jxAs6ecU{C4GL zWxKMqS=q_+p?Rbx8f>AkJCo5b>soIrw$h-G7nrEiwSBTnem*;FB7<_FQSel%ykDLg zIi1$8>Z9gOSWk%`W}aR>mCwxrJ37F5k! zp4SZu2bPz=QMtKs|MtE70{wXE!eZ63eb$UD%ViDHM>WsWn46F(3Hyf~G5DjcC}J1- zq69;d*cKF95(Q&(njSQ#4pYsEpPw2wr$IU4W2p3iIo&>)*3avj6OXtv5rby}w1aNM>cHBgV^83!~na!`uYBR?^@Yb7R!ok`5H zng7Uf_YJdNcPX|pbrjyA=d*S&11$=FG)Z=utMnrh=(tY3Tl1Ca#)lZ?w@tg-Fl&C- z9VpN21Iu;nHnYj6h6>>28dj^{6&ZJ(0&mOQ?V}C!Yk0otcY_Ra#U5kI+?54tj%~9V zd39;V%lUlb8$2kClyTXoU5i*5W`~A-V3Lw!?-95+KGh>z^7dp)^5~S=K}40?m)Z;i zQG!S?u46hS*4bm^A(d{CmkXczFAAT6gk{Oqvq6OL+xi;{-L7ZIH$=Ro}u`- z=kQ&8h7Je8WBaZ`_E}*nti8k!ClAsY{o1_NI~)6;c)H7>v!9q5c1rzBDcFV%s-HFB z`kuaXBBRgGYhUWzS@q@muclPD)z5ei-z85JDUOT?U!j@PxRvEhw-c;l>I11RJD^3c zY_??Tw3eck5_ya`uqCa9stIwo>zPevw5+y8ZO5e(O!qm=s88$#Y-xEat-n7%YRMxt zQR=@=?@ng)@WqE(MxUG0enGB| z9Egw*#|hPXM6zl-V*Ps5iZ*I9^2gzsK^}xP9XDPK)50vu>9|(gB)1VYD!D@uT&z0z zcdKFboxjMTqwxD8i9GyZQW$?EO9`R?DZsbvCUwK|xKA}G%8*$Fv1DMG7mIRVe+-BF z>;h6pVnLTRyDig=*7XM#dt?kRE@SCv&q?v4V>U;5{aE@FiuO^7;nmX`vV2yh8OE3PjT7pq-z2^0`5@*s|@HZVMq3zsI+`sUoIxh725 zD2Z2-wA0Q8yX~3m?^1f|$Ej1L>0h1vuPfCO8=zT$?q z)zYA!y&Ky6L0bPwX_RFSvo%V?Y4eRYGx~dHwcg3txadNYAFLbWEqa*n!(k&Dk__WQ zm^@%CPtU!P(a)aMz6^~elnI~bRHKNB=jH1q{muwIMt)eF4k&S-v5vtjk`zrvq*ZT% z^#txjRLEV6YC6L8hijI2J46)=wg4ai}ufsFd0=^vKPu2a6eB@e~&LrpYHdKg``9adV-Mz zl^o@Sp4UP`{gaJ*!9@n6w#RDwLa|3{F(MO9EZ#r$ifkVpX-mE+D@sP1EL?i^S&JG$ z#Rzpq4ZVKqFTzDlWOM#Yo2&+ypt`CDAMVTIxI z%c^3dww^<+^3~Htx+xJ{FpT_zSWX7lDgGZWo`!vY0oVTA_l%P7o&3q}=X{V$>&E&> zXF}u?Zg0^glce;QQYr1dKibWqgoahuFmOdpjvU{u`#GPS%jlObX}!sCNk4emRV;zx z>#!Hp`8AXuVB+hp{^e}#rwJ%DZ&}O{Qmvd<&5nMy)iUs0u@Gju}|KbV;jvxR6 zAOHd&00JNY0w4eaAOHd&Funxv{eR=DTZ{|@KmY_l00ck)1V8`;KmY_l00aa9-2WFO za0CGm009sH0T2KI5C8!X009sHf$=4P{r~vt79#@z5C8!X009sH0T2KI5C8!X00BV& z@BbGha0CGm009sH0T2KI5C8!X009sHf$=4P`~TysTZ{|@KmY_l00ck)1V8`;KmY_l z00aa9?EeJ`96s009sH0T2KI z5C8!X009sHfk6V;{|^$uIS7CN2!H?xfB*=900@8p2!H?xj2{8K|9|{+iV=YT2!H?x VfB*=900@8p2!H?xfWRPu{{>_kT=f6| literal 0 HcmV?d00001 diff --git a/management/client/client_test.go b/management/client/client_test.go index a082e354b..313a67617 100644 --- a/management/client/client_test.go +++ b/management/client/client_test.go @@ -47,25 +47,18 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) { level, _ := log.ParseLevel("debug") log.SetLevel(level) - testDir := t.TempDir() - config := &mgmt.Config{} _, err := util.ReadJson("../server/testdata/management.json", config) if err != nil { t.Fatal(err) } - config.Datadir = testDir - err = util.CopyFileContents("../server/testdata/store.json", filepath.Join(testDir, "store.json")) - if err != nil { - t.Fatal(err) - } lis, err := net.Listen("tcp", ":0") if err != nil { t.Fatal(err) } s := grpc.NewServer() - store, cleanUp, err := mgmt.NewTestStoreFromJson(context.Background(), config.Datadir) + store, cleanUp, err := NewSqliteTestStore(t, context.Background(), "../server/testdata/store.sqlite") if err != nil { t.Fatal(err) } @@ -521,3 +514,22 @@ func Test_GetPKCEAuthorizationFlow(t *testing.T) { assert.Equal(t, expectedFlowInfo.ProviderConfig.ClientID, flowInfo.ProviderConfig.ClientID, "provider configured client ID should match") assert.Equal(t, expectedFlowInfo.ProviderConfig.ClientSecret, flowInfo.ProviderConfig.ClientSecret, "provider configured client secret should match") } + +func NewSqliteTestStore(t *testing.T, ctx context.Context, testFile string) (mgmt.Store, func(), error) { + t.Helper() + dataDir := t.TempDir() + err := util.CopyFileContents(testFile, filepath.Join(dataDir, "store.db")) + if err != nil { + t.Fatal(err) + } + + store, err := mgmt.NewSqliteStore(ctx, dataDir, nil) + if err != nil { + return nil, nil, err + } + + return store, func() { + store.Close(ctx) + os.Remove(filepath.Join(dataDir, "store.db")) + }, nil +} diff --git a/management/server/account_test.go b/management/server/account_test.go index e554ae493..198775bc3 100644 --- a/management/server/account_test.go +++ b/management/server/account_test.go @@ -2366,7 +2366,7 @@ func createManager(t TB) (*DefaultAccountManager, error) { func createStore(t TB) (Store, error) { t.Helper() dataDir := t.TempDir() - store, cleanUp, err := NewTestStoreFromJson(context.Background(), dataDir) + store, cleanUp, err := NewTestStoreFromSqlite(context.Background(), "", dataDir) if err != nil { return nil, err } diff --git a/management/server/dns_test.go b/management/server/dns_test.go index e033c1a21..23941495e 100644 --- a/management/server/dns_test.go +++ b/management/server/dns_test.go @@ -210,7 +210,7 @@ func createDNSManager(t *testing.T) (*DefaultAccountManager, error) { func createDNSStore(t *testing.T) (Store, error) { t.Helper() dataDir := t.TempDir() - store, cleanUp, err := NewTestStoreFromJson(context.Background(), dataDir) + store, cleanUp, err := NewTestStoreFromSqlite(context.Background(), "", dataDir) if err != nil { return nil, err } diff --git a/management/server/file_store.go b/management/server/file_store.go index 994a4b1ee..df3e9bb77 100644 --- a/management/server/file_store.go +++ b/management/server/file_store.go @@ -2,24 +2,18 @@ package server import ( "context" - "errors" - "net" "os" "path/filepath" "strings" "sync" "time" - "github.com/netbirdio/netbird/dns" - nbgroup "github.com/netbirdio/netbird/management/server/group" - nbpeer "github.com/netbirdio/netbird/management/server/peer" - "github.com/netbirdio/netbird/management/server/posture" - "github.com/netbirdio/netbird/management/server/status" - "github.com/netbirdio/netbird/management/server/telemetry" - "github.com/netbirdio/netbird/route" "github.com/rs/xid" log "github.com/sirupsen/logrus" + nbgroup "github.com/netbirdio/netbird/management/server/group" + nbpeer "github.com/netbirdio/netbird/management/server/peer" + "github.com/netbirdio/netbird/management/server/telemetry" "github.com/netbirdio/netbird/util" ) @@ -42,167 +36,9 @@ type FileStore struct { mux sync.Mutex `json:"-"` storeFile string `json:"-"` - // sync.Mutex indexed by resource ID - resourceLocks sync.Map `json:"-"` - globalAccountLock sync.Mutex `json:"-"` - metrics telemetry.AppMetrics `json:"-"` } -func (s *FileStore) ExecuteInTransaction(ctx context.Context, f func(store Store) error) error { - return f(s) -} - -func (s *FileStore) IncrementSetupKeyUsage(ctx context.Context, setupKeyID string) error { - s.mux.Lock() - defer s.mux.Unlock() - - accountID, ok := s.SetupKeyID2AccountID[strings.ToUpper(setupKeyID)] - if !ok { - return status.NewSetupKeyNotFoundError() - } - - account, err := s.getAccount(accountID) - if err != nil { - return err - } - - account.SetupKeys[setupKeyID].UsedTimes++ - - return s.SaveAccount(ctx, account) -} - -func (s *FileStore) AddPeerToAllGroup(ctx context.Context, accountID string, peerID string) error { - s.mux.Lock() - defer s.mux.Unlock() - - account, err := s.getAccount(accountID) - if err != nil { - return err - } - - allGroup, err := account.GetGroupAll() - if err != nil || allGroup == nil { - return errors.New("all group not found") - } - - allGroup.Peers = append(allGroup.Peers, peerID) - - return nil -} - -func (s *FileStore) AddPeerToGroup(ctx context.Context, accountId string, peerId string, groupID string) error { - s.mux.Lock() - defer s.mux.Unlock() - - account, err := s.getAccount(accountId) - if err != nil { - return err - } - - account.Groups[groupID].Peers = append(account.Groups[groupID].Peers, peerId) - - return nil -} - -func (s *FileStore) AddPeerToAccount(ctx context.Context, peer *nbpeer.Peer) error { - s.mux.Lock() - defer s.mux.Unlock() - - account, ok := s.Accounts[peer.AccountID] - if !ok { - return status.NewAccountNotFoundError(peer.AccountID) - } - - account.Peers[peer.ID] = peer - return s.SaveAccount(ctx, account) -} - -func (s *FileStore) IncrementNetworkSerial(ctx context.Context, accountId string) error { - s.mux.Lock() - defer s.mux.Unlock() - - account, ok := s.Accounts[accountId] - if !ok { - return status.NewAccountNotFoundError(accountId) - } - - account.Network.Serial++ - - return s.SaveAccount(ctx, account) -} - -func (s *FileStore) GetSetupKeyBySecret(ctx context.Context, lockStrength LockingStrength, key string) (*SetupKey, error) { - s.mux.Lock() - defer s.mux.Unlock() - - accountID, ok := s.SetupKeyID2AccountID[strings.ToUpper(key)] - if !ok { - return nil, status.NewSetupKeyNotFoundError() - } - - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - setupKey, ok := account.SetupKeys[key] - if !ok { - return nil, status.Errorf(status.NotFound, "setup key not found") - } - - return setupKey, nil -} - -func (s *FileStore) GetTakenIPs(ctx context.Context, lockStrength LockingStrength, accountID string) ([]net.IP, error) { - s.mux.Lock() - defer s.mux.Unlock() - - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - var takenIps []net.IP - for _, existingPeer := range account.Peers { - takenIps = append(takenIps, existingPeer.IP) - } - - return takenIps, nil -} - -func (s *FileStore) GetPeerLabelsInAccount(ctx context.Context, lockStrength LockingStrength, accountID string) ([]string, error) { - s.mux.Lock() - defer s.mux.Unlock() - - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - existingLabels := []string{} - for _, peer := range account.Peers { - if peer.DNSLabel != "" { - existingLabels = append(existingLabels, peer.DNSLabel) - } - } - return existingLabels, nil -} - -func (s *FileStore) GetAccountNetwork(ctx context.Context, lockStrength LockingStrength, accountID string) (*Network, error) { - s.mux.Lock() - defer s.mux.Unlock() - - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - return account.Network, nil -} - -type StoredAccount struct{} - // NewFileStore restores a store from the file located in the datadir func NewFileStore(ctx context.Context, dataDir string, metrics telemetry.AppMetrics) (*FileStore, error) { fs, err := restore(ctx, filepath.Join(dataDir, storeFileName)) @@ -213,25 +49,6 @@ func NewFileStore(ctx context.Context, dataDir string, metrics telemetry.AppMetr return fs, nil } -// NewFilestoreFromSqliteStore restores a store from Sqlite and stores to Filestore json in the file located in datadir -func NewFilestoreFromSqliteStore(ctx context.Context, sqlStore *SqlStore, dataDir string, metrics telemetry.AppMetrics) (*FileStore, error) { - store, err := NewFileStore(ctx, dataDir, metrics) - if err != nil { - return nil, err - } - - err = store.SaveInstallationID(ctx, sqlStore.GetInstallationID()) - if err != nil { - return nil, err - } - - for _, account := range sqlStore.GetAllAccounts(ctx) { - store.Accounts[account.Id] = account - } - - return store, store.persist(ctx, store.storeFile) -} - // restore the state of the store from the file. // Creates a new empty store file if doesn't exist func restore(ctx context.Context, file string) (*FileStore, error) { @@ -240,7 +57,6 @@ func restore(ctx context.Context, file string) (*FileStore, error) { s := &FileStore{ Accounts: make(map[string]*Account), mux: sync.Mutex{}, - globalAccountLock: sync.Mutex{}, SetupKeyID2AccountID: make(map[string]string), PeerKeyID2AccountID: make(map[string]string), UserID2AccountID: make(map[string]string), @@ -416,252 +232,6 @@ func (s *FileStore) persist(ctx context.Context, file string) error { return nil } -// AcquireGlobalLock acquires global lock across all the accounts and returns a function that releases the lock -func (s *FileStore) AcquireGlobalLock(ctx context.Context) (unlock func()) { - log.WithContext(ctx).Debugf("acquiring global lock") - start := time.Now() - s.globalAccountLock.Lock() - - unlock = func() { - s.globalAccountLock.Unlock() - log.WithContext(ctx).Debugf("released global lock in %v", time.Since(start)) - } - - took := time.Since(start) - log.WithContext(ctx).Debugf("took %v to acquire global lock", took) - if s.metrics != nil { - s.metrics.StoreMetrics().CountGlobalLockAcquisitionDuration(took) - } - - return unlock -} - -// AcquireWriteLockByUID acquires an ID lock for writing to a resource and returns a function that releases the lock -func (s *FileStore) AcquireWriteLockByUID(ctx context.Context, uniqueID string) (unlock func()) { - log.WithContext(ctx).Debugf("acquiring lock for ID %s", uniqueID) - start := time.Now() - value, _ := s.resourceLocks.LoadOrStore(uniqueID, &sync.Mutex{}) - mtx := value.(*sync.Mutex) - mtx.Lock() - - unlock = func() { - mtx.Unlock() - log.WithContext(ctx).Debugf("released lock for ID %s in %v", uniqueID, time.Since(start)) - } - - return unlock -} - -// AcquireReadLockByUID acquires an ID lock for reading a resource and returns a function that releases the lock -// This method is still returns a write lock as file store can't handle read locks -func (s *FileStore) AcquireReadLockByUID(ctx context.Context, uniqueID string) (unlock func()) { - return s.AcquireWriteLockByUID(ctx, uniqueID) -} - -func (s *FileStore) SaveAccount(ctx context.Context, account *Account) error { - s.mux.Lock() - defer s.mux.Unlock() - - if account.Id == "" { - return status.Errorf(status.InvalidArgument, "account id should not be empty") - } - - accountCopy := account.Copy() - - s.Accounts[accountCopy.Id] = accountCopy - - // todo check that account.Id and keyId are not exist already - // because if keyId exists for other accounts this can be bad - for keyID := range accountCopy.SetupKeys { - s.SetupKeyID2AccountID[strings.ToUpper(keyID)] = accountCopy.Id - } - - // enforce peer to account index and delete peer to route indexes for rebuild - for _, peer := range accountCopy.Peers { - s.PeerKeyID2AccountID[peer.Key] = accountCopy.Id - s.PeerID2AccountID[peer.ID] = accountCopy.Id - } - - for _, user := range accountCopy.Users { - s.UserID2AccountID[user.Id] = accountCopy.Id - for _, pat := range user.PATs { - s.TokenID2UserID[pat.ID] = user.Id - s.HashedPAT2TokenID[pat.HashedToken] = pat.ID - } - } - - if accountCopy.DomainCategory == PrivateCategory && accountCopy.IsDomainPrimaryAccount { - s.PrivateDomain2AccountID[accountCopy.Domain] = accountCopy.Id - } - - return s.persist(ctx, s.storeFile) -} - -func (s *FileStore) DeleteAccount(ctx context.Context, account *Account) error { - s.mux.Lock() - defer s.mux.Unlock() - - if account.Id == "" { - return status.Errorf(status.InvalidArgument, "account id should not be empty") - } - - for keyID := range account.SetupKeys { - delete(s.SetupKeyID2AccountID, strings.ToUpper(keyID)) - } - - // enforce peer to account index and delete peer to route indexes for rebuild - for _, peer := range account.Peers { - delete(s.PeerKeyID2AccountID, peer.Key) - delete(s.PeerID2AccountID, peer.ID) - } - - for _, user := range account.Users { - for _, pat := range user.PATs { - delete(s.TokenID2UserID, pat.ID) - delete(s.HashedPAT2TokenID, pat.HashedToken) - } - delete(s.UserID2AccountID, user.Id) - } - - if account.DomainCategory == PrivateCategory && account.IsDomainPrimaryAccount { - delete(s.PrivateDomain2AccountID, account.Domain) - } - - delete(s.Accounts, account.Id) - - return s.persist(ctx, s.storeFile) -} - -// DeleteHashedPAT2TokenIDIndex removes an entry from the indexing map HashedPAT2TokenID -func (s *FileStore) DeleteHashedPAT2TokenIDIndex(hashedToken string) error { - s.mux.Lock() - defer s.mux.Unlock() - - delete(s.HashedPAT2TokenID, hashedToken) - - return nil -} - -// DeleteTokenID2UserIDIndex removes an entry from the indexing map TokenID2UserID -func (s *FileStore) DeleteTokenID2UserIDIndex(tokenID string) error { - s.mux.Lock() - defer s.mux.Unlock() - - delete(s.TokenID2UserID, tokenID) - - return nil -} - -// GetAccountByPrivateDomain returns account by private domain -func (s *FileStore) GetAccountByPrivateDomain(_ context.Context, domain string) (*Account, error) { - s.mux.Lock() - defer s.mux.Unlock() - - accountID, ok := s.PrivateDomain2AccountID[strings.ToLower(domain)] - if !ok { - return nil, status.Errorf(status.NotFound, "account not found: provided domain is not registered or is not private") - } - - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - return account.Copy(), nil -} - -// GetAccountBySetupKey returns account by setup key id -func (s *FileStore) GetAccountBySetupKey(_ context.Context, setupKey string) (*Account, error) { - s.mux.Lock() - defer s.mux.Unlock() - - accountID, ok := s.SetupKeyID2AccountID[strings.ToUpper(setupKey)] - if !ok { - return nil, status.NewSetupKeyNotFoundError() - } - - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - return account.Copy(), nil -} - -// GetTokenIDByHashedToken returns the id of a personal access token by its hashed secret -func (s *FileStore) GetTokenIDByHashedToken(_ context.Context, token string) (string, error) { - s.mux.Lock() - defer s.mux.Unlock() - - tokenID, ok := s.HashedPAT2TokenID[token] - if !ok { - return "", status.Errorf(status.NotFound, "tokenID not found: provided token doesn't exists") - } - - return tokenID, nil -} - -// GetUserByTokenID returns a User object a tokenID belongs to -func (s *FileStore) GetUserByTokenID(_ context.Context, tokenID string) (*User, error) { - s.mux.Lock() - defer s.mux.Unlock() - - userID, ok := s.TokenID2UserID[tokenID] - if !ok { - return nil, status.Errorf(status.NotFound, "user not found: provided tokenID doesn't exists") - } - - accountID, ok := s.UserID2AccountID[userID] - if !ok { - return nil, status.Errorf(status.NotFound, "accountID not found: provided userID doesn't exists") - } - - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - return account.Users[userID].Copy(), nil -} - -func (s *FileStore) GetUserByUserID(_ context.Context, _ LockingStrength, userID string) (*User, error) { - accountID, ok := s.UserID2AccountID[userID] - if !ok { - return nil, status.Errorf(status.NotFound, "accountID not found: provided userID doesn't exists") - } - - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - user := account.Users[userID].Copy() - pat := make([]PersonalAccessToken, 0, len(user.PATs)) - for _, token := range user.PATs { - if token != nil { - pat = append(pat, *token) - } - } - user.PATsG = pat - - return user, nil -} - -func (s *FileStore) GetAccountGroups(_ context.Context, accountID string) ([]*nbgroup.Group, error) { - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - groupsSlice := make([]*nbgroup.Group, 0, len(account.Groups)) - - for _, group := range account.Groups { - groupsSlice = append(groupsSlice, group) - } - - return groupsSlice, nil -} - // GetAllAccounts returns all accounts func (s *FileStore) GetAllAccounts(_ context.Context) (all []*Account) { s.mux.Lock() @@ -673,278 +243,6 @@ func (s *FileStore) GetAllAccounts(_ context.Context) (all []*Account) { return all } -// getAccount returns a reference to the Account. Should not return a copy. -func (s *FileStore) getAccount(accountID string) (*Account, error) { - account, ok := s.Accounts[accountID] - if !ok { - return nil, status.NewAccountNotFoundError(accountID) - } - - return account, nil -} - -// GetAccount returns an account for ID -func (s *FileStore) GetAccount(_ context.Context, accountID string) (*Account, error) { - s.mux.Lock() - defer s.mux.Unlock() - - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - return account.Copy(), nil -} - -// GetAccountByUser returns a user account -func (s *FileStore) GetAccountByUser(_ context.Context, userID string) (*Account, error) { - s.mux.Lock() - defer s.mux.Unlock() - - accountID, ok := s.UserID2AccountID[userID] - if !ok { - return nil, status.NewUserNotFoundError(userID) - } - - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - return account.Copy(), nil -} - -// GetAccountByPeerID returns an account for a given peer ID -func (s *FileStore) GetAccountByPeerID(ctx context.Context, peerID string) (*Account, error) { - s.mux.Lock() - defer s.mux.Unlock() - - accountID, ok := s.PeerID2AccountID[peerID] - if !ok { - return nil, status.Errorf(status.NotFound, "provided peer ID doesn't exists %s", peerID) - } - - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - // this protection is needed because when we delete a peer, we don't really remove index peerID -> accountID. - // check Account.Peers for a match - if _, ok := account.Peers[peerID]; !ok { - delete(s.PeerID2AccountID, peerID) - log.WithContext(ctx).Warnf("removed stale peerID %s to accountID %s index", peerID, accountID) - return nil, status.NewPeerNotFoundError(peerID) - } - - return account.Copy(), nil -} - -// GetAccountByPeerPubKey returns an account for a given peer WireGuard public key -func (s *FileStore) GetAccountByPeerPubKey(ctx context.Context, peerKey string) (*Account, error) { - s.mux.Lock() - defer s.mux.Unlock() - - accountID, ok := s.PeerKeyID2AccountID[peerKey] - if !ok { - return nil, status.NewPeerNotFoundError(peerKey) - } - - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - // this protection is needed because when we delete a peer, we don't really remove index peerKey -> accountID. - // check Account.Peers for a match - stale := true - for _, peer := range account.Peers { - if peer.Key == peerKey { - stale = false - break - } - } - if stale { - delete(s.PeerKeyID2AccountID, peerKey) - log.WithContext(ctx).Warnf("removed stale peerKey %s to accountID %s index", peerKey, accountID) - return nil, status.NewPeerNotFoundError(peerKey) - } - - return account.Copy(), nil -} - -func (s *FileStore) GetAccountIDByPeerPubKey(_ context.Context, peerKey string) (string, error) { - s.mux.Lock() - defer s.mux.Unlock() - - accountID, ok := s.PeerKeyID2AccountID[peerKey] - if !ok { - return "", status.NewPeerNotFoundError(peerKey) - } - - return accountID, nil -} - -func (s *FileStore) GetAccountIDByUserID(userID string) (string, error) { - s.mux.Lock() - defer s.mux.Unlock() - - accountID, ok := s.UserID2AccountID[userID] - if !ok { - return "", status.NewUserNotFoundError(userID) - } - - return accountID, nil -} - -func (s *FileStore) GetAccountIDBySetupKey(_ context.Context, setupKey string) (string, error) { - s.mux.Lock() - defer s.mux.Unlock() - - accountID, ok := s.SetupKeyID2AccountID[strings.ToUpper(setupKey)] - if !ok { - return "", status.NewSetupKeyNotFoundError() - } - - return accountID, nil -} - -func (s *FileStore) GetPeerByPeerPubKey(_ context.Context, _ LockingStrength, peerKey string) (*nbpeer.Peer, error) { - s.mux.Lock() - defer s.mux.Unlock() - - accountID, ok := s.PeerKeyID2AccountID[peerKey] - if !ok { - return nil, status.NewPeerNotFoundError(peerKey) - } - - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - for _, peer := range account.Peers { - if peer.Key == peerKey { - return peer.Copy(), nil - } - } - - return nil, status.NewPeerNotFoundError(peerKey) -} - -func (s *FileStore) GetAccountSettings(_ context.Context, _ LockingStrength, accountID string) (*Settings, error) { - s.mux.Lock() - defer s.mux.Unlock() - - account, err := s.getAccount(accountID) - if err != nil { - return nil, err - } - - return account.Settings.Copy(), nil -} - -// GetInstallationID returns the installation ID from the store -func (s *FileStore) GetInstallationID() string { - return s.InstallationID -} - -// SaveInstallationID saves the installation ID -func (s *FileStore) SaveInstallationID(ctx context.Context, ID string) error { - s.mux.Lock() - defer s.mux.Unlock() - - s.InstallationID = ID - - return s.persist(ctx, s.storeFile) -} - -// SavePeer saves the peer in the account -func (s *FileStore) SavePeer(_ context.Context, accountID string, peer *nbpeer.Peer) error { - s.mux.Lock() - defer s.mux.Unlock() - - account, err := s.getAccount(accountID) - if err != nil { - return err - } - - newPeer := peer.Copy() - - account.Peers[peer.ID] = newPeer - - s.PeerKeyID2AccountID[peer.Key] = accountID - s.PeerID2AccountID[peer.ID] = accountID - - return nil -} - -// SavePeerStatus stores the PeerStatus in memory. It doesn't attempt to persist data to speed up things. -// PeerStatus will be saved eventually when some other changes occur. -func (s *FileStore) SavePeerStatus(accountID, peerID string, peerStatus nbpeer.PeerStatus) error { - s.mux.Lock() - defer s.mux.Unlock() - - account, err := s.getAccount(accountID) - if err != nil { - return err - } - - peer := account.Peers[peerID] - if peer == nil { - return status.Errorf(status.NotFound, "peer %s not found", peerID) - } - - peer.Status = &peerStatus - - return nil -} - -// SavePeerLocation stores the PeerStatus in memory. It doesn't attempt to persist data to speed up things. -// Peer.Location will be saved eventually when some other changes occur. -func (s *FileStore) SavePeerLocation(accountID string, peerWithLocation *nbpeer.Peer) error { - s.mux.Lock() - defer s.mux.Unlock() - - account, err := s.getAccount(accountID) - if err != nil { - return err - } - - peer := account.Peers[peerWithLocation.ID] - if peer == nil { - return status.Errorf(status.NotFound, "peer %s not found", peerWithLocation.ID) - } - - peer.Location = peerWithLocation.Location - - return nil -} - -// SaveUserLastLogin stores the last login time for a user in memory. It doesn't attempt to persist data to speed up things. -func (s *FileStore) SaveUserLastLogin(_ context.Context, accountID, userID string, lastLogin time.Time) error { - s.mux.Lock() - defer s.mux.Unlock() - - account, err := s.getAccount(accountID) - if err != nil { - return err - } - - peer := account.Users[userID] - if peer == nil { - return status.Errorf(status.NotFound, "user %s not found", userID) - } - - peer.LastLogin = lastLogin - - return nil -} - -func (s *FileStore) GetPostureCheckByChecksDefinition(_ string, _ *posture.ChecksDefinition) (*posture.Checks, error) { - return nil, status.Errorf(status.Internal, "GetPostureCheckByChecksDefinition is not implemented") -} - // Close the FileStore persisting data to disk func (s *FileStore) Close(ctx context.Context) error { s.mux.Lock() @@ -959,86 +257,3 @@ func (s *FileStore) Close(ctx context.Context) error { func (s *FileStore) GetStoreEngine() StoreEngine { return FileStoreEngine } - -func (s *FileStore) SaveUsers(_ string, _ map[string]*User) error { - return status.Errorf(status.Internal, "SaveUsers is not implemented") -} - -func (s *FileStore) SaveGroups(_ string, _ map[string]*nbgroup.Group) error { - return status.Errorf(status.Internal, "SaveGroups is not implemented") -} - -func (s *FileStore) GetAccountIDByPrivateDomain(_ context.Context, _ LockingStrength, _ string) (string, error) { - return "", status.Errorf(status.Internal, "GetAccountIDByPrivateDomain is not implemented") -} - -func (s *FileStore) GetAccountDomainAndCategory(_ context.Context, _ LockingStrength, accountID string) (string, string, error) { - s.mux.Lock() - defer s.mux.Unlock() - - account, err := s.getAccount(accountID) - if err != nil { - return "", "", err - } - - return account.Domain, account.DomainCategory, nil -} - -// AccountExists checks whether an account exists by the given ID. -func (s *FileStore) AccountExists(_ context.Context, _ LockingStrength, id string) (bool, error) { - _, exists := s.Accounts[id] - return exists, nil -} - -func (s *FileStore) GetAccountDNSSettings(_ context.Context, _ LockingStrength, _ string) (*DNSSettings, error) { - return nil, status.Errorf(status.Internal, "GetAccountDNSSettings is not implemented") -} - -func (s *FileStore) GetGroupByID(_ context.Context, _ LockingStrength, _, _ string) (*nbgroup.Group, error) { - return nil, status.Errorf(status.Internal, "GetGroupByID is not implemented") -} - -func (s *FileStore) GetGroupByName(_ context.Context, _ LockingStrength, _, _ string) (*nbgroup.Group, error) { - return nil, status.Errorf(status.Internal, "GetGroupByName is not implemented") -} - -func (s *FileStore) GetAccountPolicies(_ context.Context, _ LockingStrength, _ string) ([]*Policy, error) { - return nil, status.Errorf(status.Internal, "GetPolicyByID is not implemented") -} - -func (s *FileStore) GetPolicyByID(_ context.Context, _ LockingStrength, _ string, _ string) (*Policy, error) { - return nil, status.Errorf(status.Internal, "GetPolicyByID is not implemented") - -} - -func (s *FileStore) GetAccountPostureChecks(_ context.Context, _ LockingStrength, _ string) ([]*posture.Checks, error) { - return nil, status.Errorf(status.Internal, "GetAccountPostureChecks is not implemented") -} - -func (s *FileStore) GetPostureChecksByID(_ context.Context, _ LockingStrength, _ string, _ string) (*posture.Checks, error) { - return nil, status.Errorf(status.Internal, "GetPostureChecksByID is not implemented") -} - -func (s *FileStore) GetAccountRoutes(_ context.Context, _ LockingStrength, _ string) ([]*route.Route, error) { - return nil, status.Errorf(status.Internal, "GetAccountRoutes is not implemented") -} - -func (s *FileStore) GetRouteByID(_ context.Context, _ LockingStrength, _ string, _ string) (*route.Route, error) { - return nil, status.Errorf(status.Internal, "GetRouteByID is not implemented") -} - -func (s *FileStore) GetAccountSetupKeys(_ context.Context, _ LockingStrength, _ string) ([]*SetupKey, error) { - return nil, status.Errorf(status.Internal, "GetAccountSetupKeys is not implemented") -} - -func (s *FileStore) GetSetupKeyByID(_ context.Context, _ LockingStrength, _ string, _ string) (*SetupKey, error) { - return nil, status.Errorf(status.Internal, "GetSetupKeyByID is not implemented") -} - -func (s *FileStore) GetAccountNameServerGroups(_ context.Context, _ LockingStrength, _ string) ([]*dns.NameServerGroup, error) { - return nil, status.Errorf(status.Internal, "GetAccountNameServerGroups is not implemented") -} - -func (s *FileStore) GetNameServerGroupByID(_ context.Context, _ LockingStrength, _ string, _ string) (*dns.NameServerGroup, error) { - return nil, status.Errorf(status.Internal, "GetNameServerGroupByID is not implemented") -} diff --git a/management/server/file_store_test.go b/management/server/file_store_test.go deleted file mode 100644 index 56e46b696..000000000 --- a/management/server/file_store_test.go +++ /dev/null @@ -1,655 +0,0 @@ -package server - -import ( - "context" - "crypto/sha256" - "net" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/netbirdio/netbird/management/server/group" - nbpeer "github.com/netbirdio/netbird/management/server/peer" - "github.com/netbirdio/netbird/util" -) - -type accounts struct { - Accounts map[string]*Account -} - -func TestStalePeerIndices(t *testing.T) { - storeDir := t.TempDir() - - err := util.CopyFileContents("testdata/store.json", filepath.Join(storeDir, "store.json")) - if err != nil { - t.Fatal(err) - } - - store, err := NewFileStore(context.Background(), storeDir, nil) - if err != nil { - return - } - - account, err := store.GetAccount(context.Background(), "bf1c8084-ba50-4ce7-9439-34653001fc3b") - require.NoError(t, err) - - peerID := "some_peer" - peerKey := "some_peer_key" - account.Peers[peerID] = &nbpeer.Peer{ - ID: peerID, - Key: peerKey, - } - - err = store.SaveAccount(context.Background(), account) - require.NoError(t, err) - - account.DeletePeer(peerID) - - err = store.SaveAccount(context.Background(), account) - require.NoError(t, err) - - _, err = store.GetAccountByPeerID(context.Background(), peerID) - require.Error(t, err, "expecting to get an error when found stale index") - - _, err = store.GetAccountByPeerPubKey(context.Background(), peerKey) - require.Error(t, err, "expecting to get an error when found stale index") -} - -func TestNewStore(t *testing.T) { - store := newStore(t) - defer store.Close(context.Background()) - - if store.Accounts == nil || len(store.Accounts) != 0 { - t.Errorf("expected to create a new empty Accounts map when creating a new FileStore") - } - - if store.SetupKeyID2AccountID == nil || len(store.SetupKeyID2AccountID) != 0 { - t.Errorf("expected to create a new empty SetupKeyID2AccountID map when creating a new FileStore") - } - - if store.PeerKeyID2AccountID == nil || len(store.PeerKeyID2AccountID) != 0 { - t.Errorf("expected to create a new empty PeerKeyID2AccountID map when creating a new FileStore") - } - - if store.UserID2AccountID == nil || len(store.UserID2AccountID) != 0 { - t.Errorf("expected to create a new empty UserID2AccountID map when creating a new FileStore") - } - - if store.HashedPAT2TokenID == nil || len(store.HashedPAT2TokenID) != 0 { - t.Errorf("expected to create a new empty HashedPAT2TokenID map when creating a new FileStore") - } - - if store.TokenID2UserID == nil || len(store.TokenID2UserID) != 0 { - t.Errorf("expected to create a new empty TokenID2UserID map when creating a new FileStore") - } -} - -func TestSaveAccount(t *testing.T) { - store := newStore(t) - defer store.Close(context.Background()) - - account := newAccountWithId(context.Background(), "account_id", "testuser", "") - setupKey := GenerateDefaultSetupKey() - account.SetupKeys[setupKey.Key] = setupKey - account.Peers["testpeer"] = &nbpeer.Peer{ - Key: "peerkey", - SetupKey: "peerkeysetupkey", - IP: net.IP{127, 0, 0, 1}, - Meta: nbpeer.PeerSystemMeta{}, - Name: "peer name", - Status: &nbpeer.PeerStatus{Connected: true, LastSeen: time.Now().UTC()}, - } - - // SaveAccount should trigger persist - err := store.SaveAccount(context.Background(), account) - if err != nil { - return - } - - if store.Accounts[account.Id] == nil { - t.Errorf("expecting Account to be stored after SaveAccount()") - } - - if store.PeerKeyID2AccountID["peerkey"] == "" { - t.Errorf("expecting PeerKeyID2AccountID index updated after SaveAccount()") - } - - if store.UserID2AccountID["testuser"] == "" { - t.Errorf("expecting UserID2AccountID index updated after SaveAccount()") - } - - if store.SetupKeyID2AccountID[setupKey.Key] == "" { - t.Errorf("expecting SetupKeyID2AccountID index updated after SaveAccount()") - } -} - -func TestDeleteAccount(t *testing.T) { - storeDir := t.TempDir() - storeFile := filepath.Join(storeDir, "store.json") - err := util.CopyFileContents("testdata/store.json", storeFile) - if err != nil { - t.Fatal(err) - } - - store, err := NewFileStore(context.Background(), storeDir, nil) - if err != nil { - t.Fatal(err) - } - defer store.Close(context.Background()) - - var account *Account - for _, a := range store.Accounts { - account = a - break - } - - require.NotNil(t, account, "failed to restore a FileStore file and get at least one account") - - err = store.DeleteAccount(context.Background(), account) - require.NoError(t, err, "failed to delete account, error: %v", err) - - _, ok := store.Accounts[account.Id] - require.False(t, ok, "failed to delete account") - - for id := range account.Users { - _, ok := store.UserID2AccountID[id] - assert.False(t, ok, "failed to delete UserID2AccountID index") - for _, pat := range account.Users[id].PATs { - _, ok := store.HashedPAT2TokenID[pat.HashedToken] - assert.False(t, ok, "failed to delete HashedPAT2TokenID index") - _, ok = store.TokenID2UserID[pat.ID] - assert.False(t, ok, "failed to delete TokenID2UserID index") - } - } - - for _, p := range account.Peers { - _, ok := store.PeerKeyID2AccountID[p.Key] - assert.False(t, ok, "failed to delete PeerKeyID2AccountID index") - _, ok = store.PeerID2AccountID[p.ID] - assert.False(t, ok, "failed to delete PeerID2AccountID index") - } - - for id := range account.SetupKeys { - _, ok := store.SetupKeyID2AccountID[id] - assert.False(t, ok, "failed to delete SetupKeyID2AccountID index") - } - - _, ok = store.PrivateDomain2AccountID[account.Domain] - assert.False(t, ok, "failed to delete PrivateDomain2AccountID index") - -} - -func TestStore(t *testing.T) { - store := newStore(t) - defer store.Close(context.Background()) - - account := newAccountWithId(context.Background(), "account_id", "testuser", "") - account.Peers["testpeer"] = &nbpeer.Peer{ - Key: "peerkey", - SetupKey: "peerkeysetupkey", - IP: net.IP{127, 0, 0, 1}, - Meta: nbpeer.PeerSystemMeta{}, - Name: "peer name", - Status: &nbpeer.PeerStatus{Connected: true, LastSeen: time.Now().UTC()}, - } - account.Groups["all"] = &group.Group{ - ID: "all", - Name: "all", - Peers: []string{"testpeer"}, - } - account.Policies = append(account.Policies, &Policy{ - ID: "all", - Name: "all", - Enabled: true, - Rules: []*PolicyRule{ - { - ID: "all", - Name: "all", - Sources: []string{"all"}, - Destinations: []string{"all"}, - }, - }, - }) - account.Policies = append(account.Policies, &Policy{ - ID: "dmz", - Name: "dmz", - Enabled: true, - Rules: []*PolicyRule{ - { - ID: "dmz", - Name: "dmz", - Enabled: true, - Sources: []string{"all"}, - Destinations: []string{"all"}, - }, - }, - }) - - // SaveAccount should trigger persist - err := store.SaveAccount(context.Background(), account) - if err != nil { - return - } - - restored, err := NewFileStore(context.Background(), store.storeFile, nil) - if err != nil { - return - } - - restoredAccount := restored.Accounts[account.Id] - if restoredAccount == nil { - t.Errorf("failed to restore a FileStore file - missing Account %s", account.Id) - return - } - - if restoredAccount.Peers["testpeer"] == nil { - t.Errorf("failed to restore a FileStore file - missing Peer testpeer") - } - - if restoredAccount.CreatedBy != "testuser" { - t.Errorf("failed to restore a FileStore file - missing Account CreatedBy") - } - - if restoredAccount.Users["testuser"] == nil { - t.Errorf("failed to restore a FileStore file - missing User testuser") - } - - if restoredAccount.Network == nil { - t.Errorf("failed to restore a FileStore file - missing Network") - } - - if restoredAccount.Groups["all"] == nil { - t.Errorf("failed to restore a FileStore file - missing Group all") - } - - if len(restoredAccount.Policies) != 2 { - t.Errorf("failed to restore a FileStore file - missing Policies") - return - } - - assert.Equal(t, account.Policies[0], restoredAccount.Policies[0], "failed to restore a FileStore file - missing Policy all") - assert.Equal(t, account.Policies[1], restoredAccount.Policies[1], "failed to restore a FileStore file - missing Policy dmz") -} - -func TestRestore(t *testing.T) { - storeDir := t.TempDir() - - err := util.CopyFileContents("testdata/store.json", filepath.Join(storeDir, "store.json")) - if err != nil { - t.Fatal(err) - } - - store, err := NewFileStore(context.Background(), storeDir, nil) - if err != nil { - return - } - - account := store.Accounts["bf1c8084-ba50-4ce7-9439-34653001fc3b"] - - require.NotNil(t, account, "failed to restore a FileStore file - missing account bf1c8084-ba50-4ce7-9439-34653001fc3b") - - require.NotNil(t, account.Users["edafee4e-63fb-11ec-90d6-0242ac120003"], "failed to restore a FileStore file - missing Account User edafee4e-63fb-11ec-90d6-0242ac120003") - - require.NotNil(t, account.Users["f4f6d672-63fb-11ec-90d6-0242ac120003"], "failed to restore a FileStore file - missing Account User f4f6d672-63fb-11ec-90d6-0242ac120003") - - require.NotNil(t, account.Network, "failed to restore a FileStore file - missing Account Network") - - require.NotNil(t, account.SetupKeys["A2C8E62B-38F5-4553-B31E-DD66C696CEBB"], "failed to restore a FileStore file - missing Account SetupKey A2C8E62B-38F5-4553-B31E-DD66C696CEBB") - - require.NotNil(t, account.Users["f4f6d672-63fb-11ec-90d6-0242ac120003"].PATs["9dj38s35-63fb-11ec-90d6-0242ac120003"], "failed to restore a FileStore wrong PATs length") - - require.Len(t, store.UserID2AccountID, 2, "failed to restore a FileStore wrong UserID2AccountID mapping length") - - require.Len(t, store.SetupKeyID2AccountID, 1, "failed to restore a FileStore wrong SetupKeyID2AccountID mapping length") - - require.Len(t, store.PrivateDomain2AccountID, 1, "failed to restore a FileStore wrong PrivateDomain2AccountID mapping length") - - require.Len(t, store.HashedPAT2TokenID, 1, "failed to restore a FileStore wrong HashedPAT2TokenID mapping length") - - require.Len(t, store.TokenID2UserID, 1, "failed to restore a FileStore wrong TokenID2UserID mapping length") -} - -func TestRestoreGroups_Migration(t *testing.T) { - storeDir := t.TempDir() - - err := util.CopyFileContents("testdata/store.json", filepath.Join(storeDir, "store.json")) - if err != nil { - t.Fatal(err) - } - - store, err := NewFileStore(context.Background(), storeDir, nil) - if err != nil { - return - } - - // create default group - account := store.Accounts["bf1c8084-ba50-4ce7-9439-34653001fc3b"] - account.Groups = map[string]*group.Group{ - "cfefqs706sqkneg59g3g": { - ID: "cfefqs706sqkneg59g3g", - Name: "All", - }, - } - err = store.SaveAccount(context.Background(), account) - require.NoError(t, err, "failed to save account") - - // restore account with default group with empty Issue field - if store, err = NewFileStore(context.Background(), storeDir, nil); err != nil { - return - } - account = store.Accounts["bf1c8084-ba50-4ce7-9439-34653001fc3b"] - - require.Contains(t, account.Groups, "cfefqs706sqkneg59g3g", "failed to restore a FileStore file - missing Account Groups") - require.Equal(t, group.GroupIssuedAPI, account.Groups["cfefqs706sqkneg59g3g"].Issued, "default group should has API issued mark") -} - -func TestGetAccountByPrivateDomain(t *testing.T) { - storeDir := t.TempDir() - - err := util.CopyFileContents("testdata/store.json", filepath.Join(storeDir, "store.json")) - if err != nil { - t.Fatal(err) - } - - store, err := NewFileStore(context.Background(), storeDir, nil) - if err != nil { - return - } - - existingDomain := "test.com" - - account, err := store.GetAccountByPrivateDomain(context.Background(), existingDomain) - require.NoError(t, err, "should found account") - require.Equal(t, existingDomain, account.Domain, "domains should match") - - _, err = store.GetAccountByPrivateDomain(context.Background(), "missing-domain.com") - require.Error(t, err, "should return error on domain lookup") -} - -func TestFileStore_GetAccount(t *testing.T) { - storeDir := t.TempDir() - storeFile := filepath.Join(storeDir, "store.json") - err := util.CopyFileContents("testdata/store.json", storeFile) - if err != nil { - t.Fatal(err) - } - - accounts := &accounts{} - _, err = util.ReadJson(storeFile, accounts) - if err != nil { - t.Fatal(err) - } - - store, err := NewFileStore(context.Background(), storeDir, nil) - if err != nil { - t.Fatal(err) - } - - expected := accounts.Accounts["bf1c8084-ba50-4ce7-9439-34653001fc3b"] - if expected == nil { - t.Fatalf("expected account doesn't exist") - return - } - - account, err := store.GetAccount(context.Background(), expected.Id) - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, expected.IsDomainPrimaryAccount, account.IsDomainPrimaryAccount) - assert.Equal(t, expected.DomainCategory, account.DomainCategory) - assert.Equal(t, expected.Domain, account.Domain) - assert.Equal(t, expected.CreatedBy, account.CreatedBy) - assert.Equal(t, expected.Network.Identifier, account.Network.Identifier) - assert.Len(t, account.Peers, len(expected.Peers)) - assert.Len(t, account.Users, len(expected.Users)) - assert.Len(t, account.SetupKeys, len(expected.SetupKeys)) - assert.Len(t, account.Routes, len(expected.Routes)) - assert.Len(t, account.NameServerGroups, len(expected.NameServerGroups)) -} - -func TestFileStore_GetTokenIDByHashedToken(t *testing.T) { - storeDir := t.TempDir() - storeFile := filepath.Join(storeDir, "store.json") - err := util.CopyFileContents("testdata/store.json", storeFile) - if err != nil { - t.Fatal(err) - } - - accounts := &accounts{} - _, err = util.ReadJson(storeFile, accounts) - if err != nil { - t.Fatal(err) - } - - store, err := NewFileStore(context.Background(), storeDir, nil) - if err != nil { - t.Fatal(err) - } - - hashedToken := accounts.Accounts["bf1c8084-ba50-4ce7-9439-34653001fc3b"].Users["f4f6d672-63fb-11ec-90d6-0242ac120003"].PATs["9dj38s35-63fb-11ec-90d6-0242ac120003"].HashedToken - tokenID, err := store.GetTokenIDByHashedToken(context.Background(), hashedToken) - if err != nil { - t.Fatal(err) - } - - expectedTokenID := accounts.Accounts["bf1c8084-ba50-4ce7-9439-34653001fc3b"].Users["f4f6d672-63fb-11ec-90d6-0242ac120003"].PATs["9dj38s35-63fb-11ec-90d6-0242ac120003"].ID - assert.Equal(t, expectedTokenID, tokenID) -} - -func TestFileStore_DeleteHashedPAT2TokenIDIndex(t *testing.T) { - store := newStore(t) - defer store.Close(context.Background()) - store.HashedPAT2TokenID["someHashedToken"] = "someTokenId" - - err := store.DeleteHashedPAT2TokenIDIndex("someHashedToken") - if err != nil { - t.Fatal(err) - } - - assert.Empty(t, store.HashedPAT2TokenID["someHashedToken"]) -} - -func TestFileStore_DeleteTokenID2UserIDIndex(t *testing.T) { - store := newStore(t) - store.TokenID2UserID["someTokenId"] = "someUserId" - - err := store.DeleteTokenID2UserIDIndex("someTokenId") - if err != nil { - t.Fatal(err) - } - - assert.Empty(t, store.TokenID2UserID["someTokenId"]) -} - -func TestFileStore_GetTokenIDByHashedToken_Failure(t *testing.T) { - storeDir := t.TempDir() - storeFile := filepath.Join(storeDir, "store.json") - err := util.CopyFileContents("testdata/store.json", storeFile) - if err != nil { - t.Fatal(err) - } - - accounts := &accounts{} - _, err = util.ReadJson(storeFile, accounts) - if err != nil { - t.Fatal(err) - } - - store, err := NewFileStore(context.Background(), storeDir, nil) - if err != nil { - t.Fatal(err) - } - - wrongToken := sha256.Sum256([]byte("someNotValidTokenThatFails1234")) - _, err = store.GetTokenIDByHashedToken(context.Background(), string(wrongToken[:])) - - assert.Error(t, err, "GetTokenIDByHashedToken should throw error if token invalid") -} - -func TestFileStore_GetUserByTokenID(t *testing.T) { - storeDir := t.TempDir() - storeFile := filepath.Join(storeDir, "store.json") - err := util.CopyFileContents("testdata/store.json", storeFile) - if err != nil { - t.Fatal(err) - } - - accounts := &accounts{} - _, err = util.ReadJson(storeFile, accounts) - if err != nil { - t.Fatal(err) - } - - store, err := NewFileStore(context.Background(), storeDir, nil) - if err != nil { - t.Fatal(err) - } - - tokenID := accounts.Accounts["bf1c8084-ba50-4ce7-9439-34653001fc3b"].Users["f4f6d672-63fb-11ec-90d6-0242ac120003"].PATs["9dj38s35-63fb-11ec-90d6-0242ac120003"].ID - user, err := store.GetUserByTokenID(context.Background(), tokenID) - if err != nil { - t.Fatal(err) - } - - assert.Equal(t, "f4f6d672-63fb-11ec-90d6-0242ac120003", user.Id) -} - -func TestFileStore_GetUserByTokenID_Failure(t *testing.T) { - storeDir := t.TempDir() - storeFile := filepath.Join(storeDir, "store.json") - err := util.CopyFileContents("testdata/store.json", storeFile) - if err != nil { - t.Fatal(err) - } - - accounts := &accounts{} - _, err = util.ReadJson(storeFile, accounts) - if err != nil { - t.Fatal(err) - } - - store, err := NewFileStore(context.Background(), storeDir, nil) - if err != nil { - t.Fatal(err) - } - - wrongTokenID := "someNonExistingTokenID" - _, err = store.GetUserByTokenID(context.Background(), wrongTokenID) - - assert.Error(t, err, "GetUserByTokenID should throw error if tokenID invalid") -} - -func TestFileStore_SavePeerStatus(t *testing.T) { - storeDir := t.TempDir() - - err := util.CopyFileContents("testdata/store.json", filepath.Join(storeDir, "store.json")) - if err != nil { - t.Fatal(err) - } - - store, err := NewFileStore(context.Background(), storeDir, nil) - if err != nil { - return - } - - account, err := store.getAccount("bf1c8084-ba50-4ce7-9439-34653001fc3b") - if err != nil { - t.Fatal(err) - } - - // save status of non-existing peer - newStatus := nbpeer.PeerStatus{Connected: true, LastSeen: time.Now().UTC()} - err = store.SavePeerStatus(account.Id, "non-existing-peer", newStatus) - assert.Error(t, err) - - // save new status of existing peer - account.Peers["testpeer"] = &nbpeer.Peer{ - Key: "peerkey", - ID: "testpeer", - SetupKey: "peerkeysetupkey", - IP: net.IP{127, 0, 0, 1}, - Meta: nbpeer.PeerSystemMeta{}, - Name: "peer name", - Status: &nbpeer.PeerStatus{Connected: false, LastSeen: time.Now().UTC()}, - } - - err = store.SaveAccount(context.Background(), account) - if err != nil { - t.Fatal(err) - } - - err = store.SavePeerStatus(account.Id, "testpeer", newStatus) - if err != nil { - t.Fatal(err) - } - account, err = store.getAccount(account.Id) - if err != nil { - t.Fatal(err) - } - - actual := account.Peers["testpeer"].Status - assert.Equal(t, newStatus, *actual) -} - -func TestFileStore_SavePeerLocation(t *testing.T) { - storeDir := t.TempDir() - - err := util.CopyFileContents("testdata/store.json", filepath.Join(storeDir, "store.json")) - if err != nil { - t.Fatal(err) - } - - store, err := NewFileStore(context.Background(), storeDir, nil) - if err != nil { - return - } - account, err := store.GetAccount(context.Background(), "bf1c8084-ba50-4ce7-9439-34653001fc3b") - require.NoError(t, err) - - peer := &nbpeer.Peer{ - AccountID: account.Id, - ID: "testpeer", - Location: nbpeer.Location{ - ConnectionIP: net.ParseIP("10.0.0.0"), - CountryCode: "YY", - CityName: "City", - GeoNameID: 1, - }, - Meta: nbpeer.PeerSystemMeta{}, - } - // error is expected as peer is not in store yet - err = store.SavePeerLocation(account.Id, peer) - assert.Error(t, err) - - account.Peers[peer.ID] = peer - err = store.SaveAccount(context.Background(), account) - require.NoError(t, err) - - peer.Location.ConnectionIP = net.ParseIP("35.1.1.1") - peer.Location.CountryCode = "DE" - peer.Location.CityName = "Berlin" - peer.Location.GeoNameID = 2950159 - - err = store.SavePeerLocation(account.Id, account.Peers[peer.ID]) - assert.NoError(t, err) - - account, err = store.GetAccount(context.Background(), account.Id) - require.NoError(t, err) - - actual := account.Peers[peer.ID].Location - assert.Equal(t, peer.Location, actual) -} - -func newStore(t *testing.T) *FileStore { - t.Helper() - store, err := NewFileStore(context.Background(), t.TempDir(), nil) - if err != nil { - t.Errorf("failed creating a new store") - } - - return store -} diff --git a/management/server/management_proto_test.go b/management/server/management_proto_test.go index ff09129bd..f8ab46d81 100644 --- a/management/server/management_proto_test.go +++ b/management/server/management_proto_test.go @@ -6,7 +6,6 @@ import ( "io" "net" "os" - "path/filepath" "runtime" "sync" "sync/atomic" @@ -89,14 +88,7 @@ func getServerKey(client mgmtProto.ManagementServiceClient) (*wgtypes.Key, error func Test_SyncProtocol(t *testing.T) { dir := t.TempDir() - err := util.CopyFileContents("testdata/store_with_expired_peers.json", filepath.Join(dir, "store.json")) - if err != nil { - t.Fatal(err) - } - defer func() { - os.Remove(filepath.Join(dir, "store.json")) //nolint - }() - mgmtServer, _, mgmtAddr, err := startManagementForTest(t, &Config{ + mgmtServer, _, mgmtAddr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sqlite", &Config{ Stuns: []*Host{{ Proto: "udp", URI: "stun:stun.wiretrustee.com:3468", @@ -117,6 +109,7 @@ func Test_SyncProtocol(t *testing.T) { Datadir: dir, HttpConfig: nil, }) + defer cleanup() if err != nil { t.Fatal(err) return @@ -412,18 +405,18 @@ func TestServer_GetDeviceAuthorizationFlow(t *testing.T) { } } -func startManagementForTest(t TestingT, config *Config) (*grpc.Server, *DefaultAccountManager, string, error) { +func startManagementForTest(t *testing.T, testFile string, config *Config) (*grpc.Server, *DefaultAccountManager, string, func(), error) { t.Helper() lis, err := net.Listen("tcp", "localhost:0") if err != nil { - return nil, nil, "", err + return nil, nil, "", nil, err } s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp)) - store, cleanUp, err := NewTestStoreFromJson(context.Background(), config.Datadir) + + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), testFile) if err != nil { - return nil, nil, "", err + t.Fatal(err) } - t.Cleanup(cleanUp) peersUpdateManager := NewPeersUpdateManager(nil) eventStore := &activity.InMemoryEventStore{} @@ -437,7 +430,8 @@ func startManagementForTest(t TestingT, config *Config) (*grpc.Server, *DefaultA eventStore, nil, false, MocIntegratedValidator{}, metrics) if err != nil { - return nil, nil, "", err + cleanup() + return nil, nil, "", cleanup, err } secretsManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig, config.Relay) @@ -445,7 +439,7 @@ func startManagementForTest(t TestingT, config *Config) (*grpc.Server, *DefaultA ephemeralMgr := NewEphemeralManager(store, accountManager) mgmtServer, err := NewServer(context.Background(), config, accountManager, peersUpdateManager, secretsManager, nil, ephemeralMgr) if err != nil { - return nil, nil, "", err + return nil, nil, "", cleanup, err } mgmtProto.RegisterManagementServiceServer(s, mgmtServer) @@ -455,7 +449,7 @@ func startManagementForTest(t TestingT, config *Config) (*grpc.Server, *DefaultA } }() - return s, accountManager, lis.Addr().String(), nil + return s, accountManager, lis.Addr().String(), cleanup, nil } func createRawClient(addr string) (mgmtProto.ManagementServiceClient, *grpc.ClientConn, error) { @@ -475,6 +469,7 @@ func createRawClient(addr string) (mgmtProto.ManagementServiceClient, *grpc.Clie return mgmtProto.NewManagementServiceClient(conn), conn, nil } + func Test_SyncStatusRace(t *testing.T) { if os.Getenv("CI") == "true" && os.Getenv("NETBIRD_STORE_ENGINE") == "postgres" { t.Skip("Skipping on CI and Postgres store") @@ -488,15 +483,8 @@ func Test_SyncStatusRace(t *testing.T) { func testSyncStatusRace(t *testing.T) { t.Helper() dir := t.TempDir() - err := util.CopyFileContents("testdata/store_with_expired_peers.json", filepath.Join(dir, "store.json")) - if err != nil { - t.Fatal(err) - } - defer func() { - os.Remove(filepath.Join(dir, "store.json")) //nolint - }() - mgmtServer, am, mgmtAddr, err := startManagementForTest(t, &Config{ + mgmtServer, am, mgmtAddr, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sqlite", &Config{ Stuns: []*Host{{ Proto: "udp", URI: "stun:stun.wiretrustee.com:3468", @@ -517,6 +505,7 @@ func testSyncStatusRace(t *testing.T) { Datadir: dir, HttpConfig: nil, }) + defer cleanup() if err != nil { t.Fatal(err) return @@ -665,15 +654,8 @@ func Test_LoginPerformance(t *testing.T) { t.Run(bc.name, func(t *testing.T) { t.Helper() dir := t.TempDir() - err := util.CopyFileContents("testdata/store_with_expired_peers.json", filepath.Join(dir, "store.json")) - if err != nil { - t.Fatal(err) - } - defer func() { - os.Remove(filepath.Join(dir, "store.json")) //nolint - }() - mgmtServer, am, _, err := startManagementForTest(t, &Config{ + mgmtServer, am, _, cleanup, err := startManagementForTest(t, "testdata/store_with_expired_peers.sqlite", &Config{ Stuns: []*Host{{ Proto: "udp", URI: "stun:stun.wiretrustee.com:3468", @@ -694,6 +676,7 @@ func Test_LoginPerformance(t *testing.T) { Datadir: dir, HttpConfig: nil, }) + defer cleanup() if err != nil { t.Fatal(err) return diff --git a/management/server/management_test.go b/management/server/management_test.go index 3956d96b1..ba27dc5e8 100644 --- a/management/server/management_test.go +++ b/management/server/management_test.go @@ -5,7 +5,6 @@ import ( "math/rand" "net" "os" - "path/filepath" "runtime" sync2 "sync" "time" @@ -52,8 +51,6 @@ var _ = Describe("Management service", func() { dataDir, err = os.MkdirTemp("", "wiretrustee_mgmt_test_tmp_*") Expect(err).NotTo(HaveOccurred()) - err = util.CopyFileContents("testdata/store.json", filepath.Join(dataDir, "store.json")) - Expect(err).NotTo(HaveOccurred()) var listener net.Listener config := &server.Config{} @@ -61,7 +58,7 @@ var _ = Describe("Management service", func() { Expect(err).NotTo(HaveOccurred()) config.Datadir = dataDir - s, listener = startServer(config) + s, listener = startServer(config, dataDir, "testdata/store.sqlite") addr = listener.Addr().String() client, conn = createRawClient(addr) @@ -530,12 +527,12 @@ func createRawClient(addr string) (mgmtProto.ManagementServiceClient, *grpc.Clie return mgmtProto.NewManagementServiceClient(conn), conn } -func startServer(config *server.Config) (*grpc.Server, net.Listener) { +func startServer(config *server.Config, dataDir string, testFile string) (*grpc.Server, net.Listener) { lis, err := net.Listen("tcp", ":0") Expect(err).NotTo(HaveOccurred()) s := grpc.NewServer() - store, _, err := server.NewTestStoreFromJson(context.Background(), config.Datadir) + store, _, err := server.NewTestStoreFromSqlite(context.Background(), testFile, dataDir) if err != nil { log.Fatalf("failed creating a store: %s: %v", config.Datadir, err) } diff --git a/management/server/nameserver_test.go b/management/server/nameserver_test.go index 5f8545243..7dbd4420c 100644 --- a/management/server/nameserver_test.go +++ b/management/server/nameserver_test.go @@ -773,7 +773,7 @@ func createNSManager(t *testing.T) (*DefaultAccountManager, error) { func createNSStore(t *testing.T) (Store, error) { t.Helper() dataDir := t.TempDir() - store, cleanUp, err := NewTestStoreFromJson(context.Background(), dataDir) + store, cleanUp, err := NewTestStoreFromSqlite(context.Background(), "", dataDir) if err != nil { return nil, err } diff --git a/management/server/peer_test.go b/management/server/peer_test.go index 387adb91d..225571f62 100644 --- a/management/server/peer_test.go +++ b/management/server/peer_test.go @@ -1004,7 +1004,11 @@ func Test_RegisterPeerByUser(t *testing.T) { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/extended-store.json") + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/extended-store.sqlite") + if err != nil { + t.Fatal(err) + } + defer cleanup() eventStore := &activity.InMemoryEventStore{} @@ -1065,7 +1069,11 @@ func Test_RegisterPeerBySetupKey(t *testing.T) { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/extended-store.json") + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/extended-store.sqlite") + if err != nil { + t.Fatal(err) + } + defer cleanup() eventStore := &activity.InMemoryEventStore{} @@ -1127,7 +1135,11 @@ func Test_RegisterPeerRollbackOnFailure(t *testing.T) { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/extended-store.json") + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/extended-store.sqlite") + if err != nil { + t.Fatal(err) + } + defer cleanup() eventStore := &activity.InMemoryEventStore{} diff --git a/management/server/route_test.go b/management/server/route_test.go index b556816be..fbe022102 100644 --- a/management/server/route_test.go +++ b/management/server/route_test.go @@ -1257,7 +1257,7 @@ func createRouterManager(t *testing.T) (*DefaultAccountManager, error) { func createRouterStore(t *testing.T) (Store, error) { t.Helper() dataDir := t.TempDir() - store, cleanUp, err := NewTestStoreFromJson(context.Background(), dataDir) + store, cleanUp, err := NewTestStoreFromSqlite(context.Background(), "", dataDir) if err != nil { return nil, err } @@ -1737,7 +1737,7 @@ func TestAccount_getPeersRoutesFirewall(t *testing.T) { } assert.ElementsMatch(t, routesFirewallRules, expectedRoutesFirewallRules) - //peerD is also the routing peer for route1, should contain same routes firewall rules as peerA + // peerD is also the routing peer for route1, should contain same routes firewall rules as peerA routesFirewallRules = account.getPeerRoutesFirewallRules(context.Background(), "peerD", validatedPeers) assert.Len(t, routesFirewallRules, 2) assert.ElementsMatch(t, routesFirewallRules, expectedRoutesFirewallRules) diff --git a/management/server/sql_store.go b/management/server/sql_store.go index 85c68ef44..cce748a0f 100644 --- a/management/server/sql_store.go +++ b/management/server/sql_store.go @@ -915,6 +915,28 @@ func NewPostgresqlStoreFromFileStore(ctx context.Context, fileStore *FileStore, return store, nil } +// NewPostgresqlStoreFromSqlStore restores a store from SqlStore and stores Postgres DB. +func NewPostgresqlStoreFromSqlStore(ctx context.Context, sqliteStore *SqlStore, dsn string, metrics telemetry.AppMetrics) (*SqlStore, error) { + store, err := NewPostgresqlStore(ctx, dsn, metrics) + if err != nil { + return nil, err + } + + err = store.SaveInstallationID(ctx, sqliteStore.GetInstallationID()) + if err != nil { + return nil, err + } + + for _, account := range sqliteStore.GetAllAccounts(ctx) { + err := store.SaveAccount(ctx, account) + if err != nil { + return nil, err + } + } + + return store, nil +} + func (s *SqlStore) GetSetupKeyBySecret(ctx context.Context, lockStrength LockingStrength, key string) (*SetupKey, error) { var setupKey SetupKey result := s.db.WithContext(ctx).Clauses(clause.Locking{Strength: string(lockStrength)}). diff --git a/management/server/sql_store_test.go b/management/server/sql_store_test.go index 64ef36831..dc07849d9 100644 --- a/management/server/sql_store_test.go +++ b/management/server/sql_store_test.go @@ -7,7 +7,6 @@ import ( "net" "net/netip" "os" - "path/filepath" "runtime" "testing" "time" @@ -25,7 +24,6 @@ import ( "github.com/netbirdio/netbird/management/server/status" nbpeer "github.com/netbirdio/netbird/management/server/peer" - "github.com/netbirdio/netbird/util" ) func TestSqlite_NewStore(t *testing.T) { @@ -347,7 +345,11 @@ func TestSqlite_GetAccount(t *testing.T) { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/store.json") + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/store.sqlite") + if err != nil { + t.Fatal(err) + } + defer cleanup() id := "bf1c8084-ba50-4ce7-9439-34653001fc3b" @@ -367,7 +369,11 @@ func TestSqlite_SavePeer(t *testing.T) { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/store.json") + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/store.sqlite") + if err != nil { + t.Fatal(err) + } + defer cleanup() account, err := store.GetAccount(context.Background(), "bf1c8084-ba50-4ce7-9439-34653001fc3b") require.NoError(t, err) @@ -415,7 +421,11 @@ func TestSqlite_SavePeerStatus(t *testing.T) { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/store.json") + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/store.sqlite") + defer cleanup() + if err != nil { + t.Fatal(err) + } account, err := store.GetAccount(context.Background(), "bf1c8084-ba50-4ce7-9439-34653001fc3b") require.NoError(t, err) @@ -468,8 +478,11 @@ func TestSqlite_SavePeerLocation(t *testing.T) { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/store.json") - + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/store.sqlite") + defer cleanup() + if err != nil { + t.Fatal(err) + } account, err := store.GetAccount(context.Background(), "bf1c8084-ba50-4ce7-9439-34653001fc3b") require.NoError(t, err) @@ -519,8 +532,11 @@ func TestSqlite_TestGetAccountByPrivateDomain(t *testing.T) { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/store.json") - + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/store.sqlite") + defer cleanup() + if err != nil { + t.Fatal(err) + } existingDomain := "test.com" account, err := store.GetAccountByPrivateDomain(context.Background(), existingDomain) @@ -539,8 +555,11 @@ func TestSqlite_GetTokenIDByHashedToken(t *testing.T) { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/store.json") - + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/store.sqlite") + defer cleanup() + if err != nil { + t.Fatal(err) + } hashed := "SoMeHaShEdToKeN" id := "9dj38s35-63fb-11ec-90d6-0242ac120003" @@ -560,8 +579,11 @@ func TestSqlite_GetUserByTokenID(t *testing.T) { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/store.json") - + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/store.sqlite") + defer cleanup() + if err != nil { + t.Fatal(err) + } id := "9dj38s35-63fb-11ec-90d6-0242ac120003" user, err := store.GetUserByTokenID(context.Background(), id) @@ -668,24 +690,9 @@ func newSqliteStore(t *testing.T) *SqlStore { t.Helper() store, err := NewSqliteStore(context.Background(), t.TempDir(), nil) - require.NoError(t, err) - require.NotNil(t, store) - - return store -} - -func newSqliteStoreFromFile(t *testing.T, filename string) *SqlStore { - t.Helper() - - storeDir := t.TempDir() - - err := util.CopyFileContents(filename, filepath.Join(storeDir, "store.json")) - require.NoError(t, err) - - fStore, err := NewFileStore(context.Background(), storeDir, nil) - require.NoError(t, err) - - store, err := NewSqliteStoreFromFileStore(context.Background(), fStore, storeDir, nil) + t.Cleanup(func() { + store.Close(context.Background()) + }) require.NoError(t, err) require.NotNil(t, store) @@ -733,32 +740,31 @@ func newPostgresqlStore(t *testing.T) *SqlStore { return store } -func newPostgresqlStoreFromFile(t *testing.T, filename string) *SqlStore { +func newPostgresqlStoreFromSqlite(t *testing.T, filename string) *SqlStore { t.Helper() - storeDir := t.TempDir() - err := util.CopyFileContents(filename, filepath.Join(storeDir, "store.json")) - require.NoError(t, err) + store, cleanUpQ, err := NewSqliteTestStore(context.Background(), t.TempDir(), filename) + t.Cleanup(cleanUpQ) + if err != nil { + return nil + } - fStore, err := NewFileStore(context.Background(), storeDir, nil) - require.NoError(t, err) - - cleanUp, err := testutil.CreatePGDB() + cleanUpP, err := testutil.CreatePGDB() if err != nil { t.Fatal(err) } - t.Cleanup(cleanUp) + t.Cleanup(cleanUpP) postgresDsn, ok := os.LookupEnv(postgresDsnEnv) if !ok { t.Fatalf("could not initialize postgresql store: %s is not set", postgresDsnEnv) } - store, err := NewPostgresqlStoreFromFileStore(context.Background(), fStore, postgresDsn, nil) + pstore, err := NewPostgresqlStoreFromSqlStore(context.Background(), store, postgresDsn, nil) require.NoError(t, err) require.NotNil(t, store) - return store + return pstore } func TestPostgresql_NewStore(t *testing.T) { @@ -924,7 +930,7 @@ func TestPostgresql_SavePeerStatus(t *testing.T) { t.Skipf("The PostgreSQL store is not properly supported by %s yet", runtime.GOOS) } - store := newPostgresqlStoreFromFile(t, "testdata/store.json") + store := newPostgresqlStoreFromSqlite(t, "testdata/store.sqlite") account, err := store.GetAccount(context.Background(), "bf1c8084-ba50-4ce7-9439-34653001fc3b") require.NoError(t, err) @@ -963,7 +969,7 @@ func TestPostgresql_TestGetAccountByPrivateDomain(t *testing.T) { t.Skipf("The PostgreSQL store is not properly supported by %s yet", runtime.GOOS) } - store := newPostgresqlStoreFromFile(t, "testdata/store.json") + store := newPostgresqlStoreFromSqlite(t, "testdata/store.sqlite") existingDomain := "test.com" @@ -980,7 +986,7 @@ func TestPostgresql_GetTokenIDByHashedToken(t *testing.T) { t.Skipf("The PostgreSQL store is not properly supported by %s yet", runtime.GOOS) } - store := newPostgresqlStoreFromFile(t, "testdata/store.json") + store := newPostgresqlStoreFromSqlite(t, "testdata/store.sqlite") hashed := "SoMeHaShEdToKeN" id := "9dj38s35-63fb-11ec-90d6-0242ac120003" @@ -995,7 +1001,7 @@ func TestPostgresql_GetUserByTokenID(t *testing.T) { t.Skipf("The PostgreSQL store is not properly supported by %s yet", runtime.GOOS) } - store := newPostgresqlStoreFromFile(t, "testdata/store.json") + store := newPostgresqlStoreFromSqlite(t, "testdata/store.sqlite") id := "9dj38s35-63fb-11ec-90d6-0242ac120003" @@ -1009,12 +1015,15 @@ func TestSqlite_GetTakenIPs(t *testing.T) { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/extended-store.json") - defer store.Close(context.Background()) + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/extended-store.sqlite") + defer cleanup() + if err != nil { + t.Fatal(err) + } existingAccountID := "bf1c8084-ba50-4ce7-9439-34653001fc3b" - _, err := store.GetAccount(context.Background(), existingAccountID) + _, err = store.GetAccount(context.Background(), existingAccountID) require.NoError(t, err) takenIPs, err := store.GetTakenIPs(context.Background(), LockingStrengthShare, existingAccountID) @@ -1054,12 +1063,15 @@ func TestSqlite_GetPeerLabelsInAccount(t *testing.T) { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/extended-store.json") - defer store.Close(context.Background()) + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/extended-store.sqlite") + if err != nil { + return + } + t.Cleanup(cleanup) existingAccountID := "bf1c8084-ba50-4ce7-9439-34653001fc3b" - _, err := store.GetAccount(context.Background(), existingAccountID) + _, err = store.GetAccount(context.Background(), existingAccountID) require.NoError(t, err) labels, err := store.GetPeerLabelsInAccount(context.Background(), LockingStrengthShare, existingAccountID) @@ -1096,12 +1108,15 @@ func TestSqlite_GetAccountNetwork(t *testing.T) { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/extended-store.json") - defer store.Close(context.Background()) + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/extended-store.sqlite") + t.Cleanup(cleanup) + if err != nil { + t.Fatal(err) + } existingAccountID := "bf1c8084-ba50-4ce7-9439-34653001fc3b" - _, err := store.GetAccount(context.Background(), existingAccountID) + _, err = store.GetAccount(context.Background(), existingAccountID) require.NoError(t, err) network, err := store.GetAccountNetwork(context.Background(), LockingStrengthShare, existingAccountID) @@ -1118,12 +1133,15 @@ func TestSqlite_GetSetupKeyBySecret(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/extended-store.json") - defer store.Close(context.Background()) + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/extended-store.sqlite") + t.Cleanup(cleanup) + if err != nil { + t.Fatal(err) + } existingAccountID := "bf1c8084-ba50-4ce7-9439-34653001fc3b" - _, err := store.GetAccount(context.Background(), existingAccountID) + _, err = store.GetAccount(context.Background(), existingAccountID) require.NoError(t, err) setupKey, err := store.GetSetupKeyBySecret(context.Background(), LockingStrengthShare, "A2C8E62B-38F5-4553-B31E-DD66C696CEBB") @@ -1137,12 +1155,16 @@ func TestSqlite_incrementSetupKeyUsage(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("The SQLite store is not properly supported by Windows yet") } - store := newSqliteStoreFromFile(t, "testdata/extended-store.json") - defer store.Close(context.Background()) + + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/extended-store.sqlite") + t.Cleanup(cleanup) + if err != nil { + t.Fatal(err) + } existingAccountID := "bf1c8084-ba50-4ce7-9439-34653001fc3b" - _, err := store.GetAccount(context.Background(), existingAccountID) + _, err = store.GetAccount(context.Background(), existingAccountID) require.NoError(t, err) setupKey, err := store.GetSetupKeyBySecret(context.Background(), LockingStrengthShare, "A2C8E62B-38F5-4553-B31E-DD66C696CEBB") diff --git a/management/server/store.go b/management/server/store.go index f34a73c2d..041c936ae 100644 --- a/management/server/store.go +++ b/management/server/store.go @@ -12,10 +12,11 @@ import ( "strings" "time" - "github.com/netbirdio/netbird/dns" log "github.com/sirupsen/logrus" "gorm.io/gorm" + "github.com/netbirdio/netbird/dns" + nbgroup "github.com/netbirdio/netbird/management/server/group" "github.com/netbirdio/netbird/management/server/telemetry" @@ -236,23 +237,29 @@ func getMigrations(ctx context.Context) []migrationFunc { } } -// NewTestStoreFromJson is only used in tests -func NewTestStoreFromJson(ctx context.Context, dataDir string) (Store, func(), error) { - fstore, err := NewFileStore(ctx, dataDir, nil) - if err != nil { - return nil, nil, err - } - +// NewTestStoreFromSqlite is only used in tests +func NewTestStoreFromSqlite(ctx context.Context, filename string, dataDir string) (Store, func(), error) { // if store engine is not set in the config we first try to evaluate NETBIRD_STORE_ENGINE kind := getStoreEngineFromEnv() if kind == "" { kind = SqliteStoreEngine } - var ( - store Store - cleanUp func() - ) + var store *SqlStore + var err error + var cleanUp func() + + if filename == "" { + store, err = NewSqliteStore(ctx, dataDir, nil) + cleanUp = func() { + store.Close(ctx) + } + } else { + store, cleanUp, err = NewSqliteTestStore(ctx, dataDir, filename) + } + if err != nil { + return nil, nil, err + } if kind == PostgresStoreEngine { cleanUp, err = testutil.CreatePGDB() @@ -265,21 +272,32 @@ func NewTestStoreFromJson(ctx context.Context, dataDir string) (Store, func(), e return nil, nil, fmt.Errorf("%s is not set", postgresDsnEnv) } - store, err = NewPostgresqlStoreFromFileStore(ctx, fstore, dsn, nil) + store, err = NewPostgresqlStoreFromSqlStore(ctx, store, dsn, nil) if err != nil { return nil, nil, err } - } else { - store, err = NewSqliteStoreFromFileStore(ctx, fstore, dataDir, nil) - if err != nil { - return nil, nil, err - } - cleanUp = func() { store.Close(ctx) } } return store, cleanUp, nil } +func NewSqliteTestStore(ctx context.Context, dataDir string, testFile string) (*SqlStore, func(), error) { + err := util.CopyFileContents(testFile, filepath.Join(dataDir, "store.db")) + if err != nil { + return nil, nil, err + } + + store, err := NewSqliteStore(ctx, dataDir, nil) + if err != nil { + return nil, nil, err + } + + return store, func() { + store.Close(ctx) + os.Remove(filepath.Join(dataDir, "store.db")) + }, nil +} + // MigrateFileStoreToSqlite migrates the file store to the SQLite store. func MigrateFileStoreToSqlite(ctx context.Context, dataDir string) error { fileStorePath := path.Join(dataDir, storeFileName) diff --git a/management/server/store_test.go b/management/server/store_test.go index 40c36c9e0..fc821670d 100644 --- a/management/server/store_test.go +++ b/management/server/store_test.go @@ -14,12 +14,6 @@ type benchCase struct { size int } -var newFs = func(b *testing.B) Store { - b.Helper() - store, _ := NewFileStore(context.Background(), b.TempDir(), nil) - return store -} - var newSqlite = func(b *testing.B) Store { b.Helper() store, _ := NewSqliteStore(context.Background(), b.TempDir(), nil) @@ -28,13 +22,9 @@ var newSqlite = func(b *testing.B) Store { func BenchmarkTest_StoreWrite(b *testing.B) { cases := []benchCase{ - {name: "FileStore_Write", storeFn: newFs, size: 100}, {name: "SqliteStore_Write", storeFn: newSqlite, size: 100}, - {name: "FileStore_Write", storeFn: newFs, size: 500}, {name: "SqliteStore_Write", storeFn: newSqlite, size: 500}, - {name: "FileStore_Write", storeFn: newFs, size: 1000}, {name: "SqliteStore_Write", storeFn: newSqlite, size: 1000}, - {name: "FileStore_Write", storeFn: newFs, size: 2000}, {name: "SqliteStore_Write", storeFn: newSqlite, size: 2000}, } @@ -61,11 +51,8 @@ func BenchmarkTest_StoreWrite(b *testing.B) { func BenchmarkTest_StoreRead(b *testing.B) { cases := []benchCase{ - {name: "FileStore_Read", storeFn: newFs, size: 100}, {name: "SqliteStore_Read", storeFn: newSqlite, size: 100}, - {name: "FileStore_Read", storeFn: newFs, size: 500}, {name: "SqliteStore_Read", storeFn: newSqlite, size: 500}, - {name: "FileStore_Read", storeFn: newFs, size: 1000}, {name: "SqliteStore_Read", storeFn: newSqlite, size: 1000}, } @@ -89,3 +76,11 @@ func BenchmarkTest_StoreRead(b *testing.B) { }) } } + +func newStore(t *testing.T) Store { + t.Helper() + + store := newSqliteStore(t) + + return store +} diff --git a/management/server/testdata/extended-store.json b/management/server/testdata/extended-store.json deleted file mode 100644 index 7f96e57a8..000000000 --- a/management/server/testdata/extended-store.json +++ /dev/null @@ -1,120 +0,0 @@ -{ - "Accounts": { - "bf1c8084-ba50-4ce7-9439-34653001fc3b": { - "Id": "bf1c8084-ba50-4ce7-9439-34653001fc3b", - "CreatedBy": "", - "Domain": "test.com", - "DomainCategory": "private", - "IsDomainPrimaryAccount": true, - "SetupKeys": { - "A2C8E62B-38F5-4553-B31E-DD66C696CEBB": { - "Id": "A2C8E62B-38F5-4553-B31E-DD66C696CEBB", - "AccountID": "", - "Key": "A2C8E62B-38F5-4553-B31E-DD66C696CEBB", - "Name": "Default key", - "Type": "reusable", - "CreatedAt": "2021-08-19T20:46:20.005936822+02:00", - "ExpiresAt": "2321-09-18T20:46:20.005936822+02:00", - "UpdatedAt": "0001-01-01T00:00:00Z", - "Revoked": false, - "UsedTimes": 0, - "LastUsed": "0001-01-01T00:00:00Z", - "AutoGroups": ["cfefqs706sqkneg59g2g"], - "UsageLimit": 0, - "Ephemeral": false - }, - "A2C8E62B-38F5-4553-B31E-DD66C696CEBC": { - "Id": "A2C8E62B-38F5-4553-B31E-DD66C696CEBC", - "AccountID": "", - "Key": "A2C8E62B-38F5-4553-B31E-DD66C696CEBC", - "Name": "Faulty key with non existing group", - "Type": "reusable", - "CreatedAt": "2021-08-19T20:46:20.005936822+02:00", - "ExpiresAt": "2321-09-18T20:46:20.005936822+02:00", - "UpdatedAt": "0001-01-01T00:00:00Z", - "Revoked": false, - "UsedTimes": 0, - "LastUsed": "0001-01-01T00:00:00Z", - "AutoGroups": ["abcd"], - "UsageLimit": 0, - "Ephemeral": false - } - }, - "Network": { - "id": "af1c8024-ha40-4ce2-9418-34653101fc3c", - "Net": { - "IP": "100.64.0.0", - "Mask": "//8AAA==" - }, - "Dns": "", - "Serial": 0 - }, - "Peers": {}, - "Users": { - "edafee4e-63fb-11ec-90d6-0242ac120003": { - "Id": "edafee4e-63fb-11ec-90d6-0242ac120003", - "AccountID": "", - "Role": "admin", - "IsServiceUser": false, - "ServiceUserName": "", - "AutoGroups": ["cfefqs706sqkneg59g3g"], - "PATs": {}, - "Blocked": false, - "LastLogin": "0001-01-01T00:00:00Z" - }, - "f4f6d672-63fb-11ec-90d6-0242ac120003": { - "Id": "f4f6d672-63fb-11ec-90d6-0242ac120003", - "AccountID": "", - "Role": "user", - "IsServiceUser": false, - "ServiceUserName": "", - "AutoGroups": null, - "PATs": { - "9dj38s35-63fb-11ec-90d6-0242ac120003": { - "ID": "9dj38s35-63fb-11ec-90d6-0242ac120003", - "UserID": "", - "Name": "", - "HashedToken": "SoMeHaShEdToKeN", - "ExpirationDate": "2023-02-27T00:00:00Z", - "CreatedBy": "user", - "CreatedAt": "2023-01-01T00:00:00Z", - "LastUsed": "2023-02-01T00:00:00Z" - } - }, - "Blocked": false, - "LastLogin": "0001-01-01T00:00:00Z" - } - }, - "Groups": { - "cfefqs706sqkneg59g4g": { - "ID": "cfefqs706sqkneg59g4g", - "Name": "All", - "Peers": [] - }, - "cfefqs706sqkneg59g3g": { - "ID": "cfefqs706sqkneg59g3g", - "Name": "AwesomeGroup1", - "Peers": [] - }, - "cfefqs706sqkneg59g2g": { - "ID": "cfefqs706sqkneg59g2g", - "Name": "AwesomeGroup2", - "Peers": [] - } - }, - "Rules": null, - "Policies": [], - "Routes": null, - "NameServerGroups": null, - "DNSSettings": null, - "Settings": { - "PeerLoginExpirationEnabled": false, - "PeerLoginExpiration": 86400000000000, - "GroupsPropagationEnabled": false, - "JWTGroupsEnabled": false, - "JWTGroupsClaimName": "" - } - } - }, - "InstallationID": "" -} diff --git a/management/server/testdata/extended-store.sqlite b/management/server/testdata/extended-store.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..81aea8118ccf7d3af562ddece1f3007f1e8fc942 GIT binary patch literal 163840 zcmeI5Piz}ke#beYMN+aQ#_=W-C(-UmiDMI%Y*LhD$qU1kW!f=p%ZX%R*D!+NhyTukw&U@-<(bEEZE!smbz2#6Km;T zzyElfJ{<86=v@kZ9`y0u#5wD4N4aqJTQ2iP_S^B_9lJk%F*Q1#9sPQAGr2Q*CV52C zM!rt`KJg{}qsQ6TQ}X4LQbISX?2-MTK@C*3wquH8wQia1maP`t_Y<9|*V6LGD^ll@ z$kwZm)E2XB)6g2KR;e)CRvmMf8MfNCnWgI0_}t@b8>Q8|CFREY)zVLu^1;=Vl{@Q7 zIli*;nX-4Xa_;fwWLjQXkv?B_wCx6qFA$%a_1biI^~$Z%z;h{QsI_IqVUL{I;I!VM zp*5MGwWHZPtm?w>)7YbyZfTBg8fsN@NR+=idXvqTD;A^ARdsvc7xq&$CoI>{Y=@(z zCIp2)+PHChb>p6LvvkjGU2b;m&idxvjny0Lca?H&SLni4TbkQj5wa`H<4aB*C<3PwX|EloO^QVXj+zK>FGUT20}!<9E&R1EgR8ks2WeLme~l@ zs@p0xc2BP`0ky0c?IyGI%52%-T4fC;P8doLi>qM^;bY7jUcSz^8)jvf>o%xW=(}Oo zbt9zkz*?fVq1$$w7v{F9TrFK&-MV#GnbBJM%xqaV99H*i&tf%#TvkLHvtlws!p?rn z@BG1b=CN}Je~hVx*ybYE<`?u0E_W;?Uy=tIoY0zD5N{LP9jD;Up74-MOBD;GEw6cY2^cq zQ=G!}MmBTWEj5&&w|v8Dl3A)drtO3cP$bq()wCmGyUa3JBa&1xE#C;rqExG)IXs9( zqUDGN~+v+LLj6K~ljV3e5LAweNYOnu1c6QA= zluMDr+E*)PH89j}rta)V^c&>VnJF?N?(jL>oEV|~5p{SsmwA?(b8DwCN~36OFO`-r zPfMMXfe#9XdpLA<6Vv@pshb%JwyuNfX7#wfCm$S1%hS`+Q&~8xu3Y!iXw+?WGoHhD zg$GI~*6K`c z=uMr@9h8nS-RCf)F0mJ|rQ&ExzCJxDWQI;r=zdOZC)09qO6o{~Rr!%vt(r|uH~hmR z{nq4Xd6KRj`H5sjZUyDs7avJ!d1^}f1-UwMAijkEJ`%MaKRIeU{$%)|>9KcC&mX%- zDtQoAH?3f1O>;Oer)B9)jogNJ(vUm!gNt4z|E|~cuJiYE=-v1GeiC{3-Xu5vge-+e z0aAdY8+B@iZVR8PQj{UH;<2P>Mi__+Uq6JyeRhFRLjpmI)!Plt@)ikuI(w-0&qD)g zZ#_uyqovgcdHq28Q;POpiT>CSCsk=^dNbmzBT`zUVLlv$h^D=kM$1tOWPX5)30gsY zuDCxjCYL|~Q6~>#TG12q59HkWk(9hPHE68)Caf0xPf5PcP9%OXIYF;d_G*Hjcz^&1 zfB*=900@8p2!H?xfB*=900=yTKxajIV{78})H`EaKbgoJ&x~d=8NROGt`#cF`Q?SV zZEZ0>w@_gh=2jMpD|5w#rNv@CU#L}z+w=MPg}Fk0EtFu*P9iZ*?Ub^AO3)Jz5C8!X009sH0T2KI5C8!X z009sHfme*c>EzbfF#8Yg`v03m_M2C%gJ>=YfB*=900@8p2!H?xfB*=900@ADm`X{~f+ZJ3L{!_;$4Yd4&I zg}%f`d8j)(3Vo}T!XD{1|Gof4{3X4`=qo7bdlcwa_QG6#d9JXc%;zsIEM1(Zd(ZQW zE5)Vd`FZzl_W2?gS(z&=4;DGNmOR}aKPR3_K2P_^)AOu&y+5OESE@6gJU%0FV()px zE)7NO%6CNU%2ih5%s8@_2~DNOY7guS`6c_ouEFYyEA{#M%qRT%|48;IQQt!M)XfB*=900@8p z2!H?xfB*=900=|~@b&*N_bG_=|1g65{(t(>KRiGH1V8`;KmY_l00ck)1V8`;KmY^| zF#&%4AM5`^tYCBv1V8`;KmY_l00ck)1V8`;KmY`Y0M`F70}ucK5C8!X009sH0T2KI z5C8!XIQ#^#{y+RWMh`&%1V8`;KmY_l00ck)1V8`;KtTNd-{e0fvg`DM2MB-w2!H?x zfB*=900@8p2!H?xygURtZ;ecDZM~7nOdQW#zI`GyQCp}jRhKTz&n*>e+jE5itIVzB zt4nkF`Gt9{QkbXT3oH)vU3hJqS@%B~`0Il7`-Tg1g*;WO6qYXL3m1#a=jIFfr3=gM zj~i+&y>niorpeT_OK#IxRjV>rc;rAjf#`pifJcf;yfB*=9 z00@8p2!H?xfB*=900@A9M}U9-|1jGf;`{%H*?hqIAGQSoAOHd&00JNY0w4eaAOHd& z00J)^f$!z_|D^0X-T(j6^%V^Q0T2KI5C8!X009sH0T2KI5CDOflmOlTKen|sarW{X zmoKkWKP@iX#l>gc`ajI2{7v&V`%v55DOK;9H`zM<6+)5z1YvIef|Ac)6wkBbmH+jE zn^~C47kV=NQhgcx`oEO@Q-Yp&fB*=900@8p2!H?xfB*=900@8p2)tqhPA9jQsCyr+(`oC#^zCO%v2&_J2w%KIYEwkO4r(XoP z|H;P4;5v)XU1vem`JqI0o``Sf!f!3m00ck) z1V8`;KmY_l00cnbl_Bs}a#4!gp-9GO6zd~$a&fG`#6o={``?L4={JdE-(>&g*ssQa zckKT7#nk9{cJ%Af&E#Jvcale>-;8{n_b11|@ru;BB(nADBelgW+cdO>s#Pk?wpGX6WrnTtFE3E_s;hwS z_Z;ypIKH+~TD@CRZmeG|{ZuI*+|;siXI&}BS5`h#_BvF~J>Hy5%PT9==c|sk-C*$r z;#0F;o9?b&xm6P9`c5m0(L-4|LtQT`4twOx27T}j4Xw%itR2nXVO3Xqej0n!(k;!= zO+&3}4vF$tM{ly(a>ZiwxvFmO`@(*T=7i-Mn(c5~pe6)`Kiar)dv)WUaLZC!45 z?aun<-Hp{7>vxrMZC4nNt+q7PvDJE6xprrxbmRKES0GS?vQfHL+9<8Bl{OX8HJj8# zoz3O8C(`nwEOkZ$E9Hf3ZlY$}aL6vj@npY=^vbzLAyTRGZh1P%v6OsC9yD&;dJ5ug z;>o3>X<3$~r}u=#h(z&nENTxTEvm8589k+0W+PCVZmZOr`hrBo1eT{B6%*rm;Tu`gfX2Y!OMo683Ekrf1+jg55=9Z^iEnQpP zx^-8Xp)Jap*|KgptnOKs#cHIFvLe!$6_XhfcJ^C-=MT0bkDcp_Vhkz7HW#razhGca z5GMMwww02dkTD2&) zv1l*secNG8J_26z8nH6b=C(yFYYP;e@hMVZ3m*3Qkc1^1|ZL6m|Gxl`LG@8sH zpXw?=l&k5DZd1azs}y(}+IAN$*RN_jn$zYP)y1s=fa6aMv~KP%cFdYhSIH)xc1@nYyzd(QlAbXQs%A__*irU1Egx2f?9z zS1$9cFy+=>5N~69skD4~TI!q(d{8jl!=baAnC^E<-ONz1bsbbUtH<>{`QS)eo}QMT z%EDQ7<+`6nqi(C4@f^M@JWxV$WI*_G&7Arjna;FZd=^vfNwwG>O?ty-OQue9DViyf z#|Q#j++3&}6L;ITR%dEMZ|ZdJpmdDsK8G1~iM@a=6-QI@_31%Vp3o@@-Os7*WLhpx zNgXM$DnGocRkNw-hJSdZ-trWH(vX=3K(v@E@;k=yW28ghqzaM7#e-}Rc_b^d-1 zz59OOPa+TBo8-oykfrb_KnielqfX7xZQ)Z@iZWzYJeKrK^8!)f>xXc-&n^&ZNFZpj zdb^=n-n@QKXAjl>$z>qztvM-vw6yvluOCQ%O3~gc(H|S)q$&+fZ$_MTL`rKk%!h*z z(OlNjXvs@~%nxueK`W@w758Vmf}L8D|&+dft))(l9Jb^28}h}gw=xoDM>5s z3}0NA4mQ@IeFx(k-ty;+lZF{YjB?f!He1D1(bpZVWQCHCIR4~>;E9EM)E0*5k%i?gh z2PPPvE+AjRa|5B_@A`*8JkEACtFq(y5xd{a`i-V{8SUGKI|1QKzpmo?merw7KYKN_ z_QRC?lfod&>}RVM`qQRs@22GoC#B9t`t;UKRx(7;@H zFEF(@`_8j=A-(Dg?)wivJDHYOPf4Arz<63_L$BRA~+fg{XTgY2Y zhbknK3aTvUo=9?9K6Og^qU0{ELxSN7-;P>WI4kOGx>b8q3O-tVbBM8)evQ)7G31M= z2t|GoHUsBkn{A8q?AN-jS80tVN<~&e6mcE1Vg?KImT5UbA%37*no$qxIoJV-zUibA zp_tx{yl&5<4#zJ=-1%-E-cgR+_lxswbiSo+=aJN^379&T##3iRqKwuCb7me+D_mNO&a&IicsZ z5Lf?X<2K)9P%AsEvg<4MU@dxNyn!X!7u}KVq9bjEH)Z{jo+fkW-+0!f#;2mXI-?W4 zb@VUYNlooUZ@B22f4B*6_Uo)q($$#f2cw(1&vl{V@%zU`Z29~>pV{nD(Q!XaCvfrVORM$ zIg^%iIq8XaAu-~eqZOZu_BWatwTZA1#7rMT4e&X|1UxNka&Oq z2!H?xfB*=900@8p2!H?xfB*;_UIP63Ki2<;SHI{P2!H?xfB*=900@8p2!H?xfB*<^ z0$BgU4nP0|KmY_l00ck)1V8`;KmY_l;P4Z`{r`tw$LJvlfB*=900@8p2!H?xfB*=9 z00`jvKWqR5KmY_l00ck)1V8`;KmY_l00a&{0j&QIzmCyE5C8!X009sH0T2KI5C8!X z009ud{r|855C8!X009sH0T2KI5C8!X009s<`~-0Q|M2S=Jp=&|009sH0T2KI5C8!X z009sH0j&RF10VnbAOHd&00JNY0w4eaAOHd&aQF$}{{O?TWAqRNKmY_l00ck)1V8`; zKmY_l00eOTA2t92AOHd&00JNY0w4eaAOHd&00M`f0M`G9U&rVn2!H?xfB*=900@8p z2!H?xfB*>a`~Rivw+VXU0RkWZ0w4eaAOHd&00JNY0w4eaAaM8yB&B3x0)PL1_!NsC zfdB}A00@8p2!H?xfB*=900@9UF9EFodx_u}1V8`;KmY_l00ck)1V8`;KmY^|9|8RR z|KZaqdISO>00JNY0w4eaAOHd&00JNY0=)#V{_iD%V-NrV5C8!X009sH0T2KI5C8!X lID7plVC(-UmiDMI%Y*Li=$@Av+Fe!utoy*D$Iwmy1bdQ7X>PTladeBzaaq9p!8(-MiqJM{lD{THvd z=))OtLPtgV-0$N%iL1`vjrhXp?|qrq)8CK(?%c=YtI3h^^vE|OTf=)Jmxs?N#?Uv3 z-zUDNfAqTYYEr#XnrG(TdW#!L!Zs zQ@z0)*S3tBZj?&Qb#>3)XO^osUFPU!xp(gQ_07WCc0s$hal7zyt$2JjMeWgsR_t9_ z@p3V2teAPWHJMUZR+KN+JY%=UdKc)OnhyK4y>{zCp>JE-W$JBF^Vm~wHflCJwTwCw zS$l@N$I1Z=k;a}jOvms{+tSO1N230_XV%$lvE(rNT-J9Fg|J95JXx-0xSo%ex)2rq zaP!{7wav%c{lep*cbVDsM;lw)n``$rwzXnqpX%fwVems+KkaKXJ(70<*{mLdk(7*G6JblR&&uKOytWREittuj`?YqAGYjqjBTpX~koVlQ$7~ zBbRy2hAt(T4PjVyGE05Wc0Jhud1BSpZ8s*i&m4=@Vo4?25k^qtr5ZKE^MhDSs-qs; z)emT7a-4z*knSx@?*lnZr z^{cy{;WhmX^6M^Rv1u*~lx)jlB?=LebCC17kI;F9=_})~&zdH&(v1cU`@kS2M~=py zOx!Dkw&d=~mgLeYv;2rEw=cCBM716w{Gg@|$$B1HJuZ2~yM~H5^MwRI`vu;w` zQRyD0dkQmZ6MF$$%8w+~yVLzbrqo2P{W-ZioKo{sN=u2XN&50#WUHKqKDTpc+OAt8=KaqAJuaoZ7#;p3LaVVj{p4$gG)AgpRT z(aM_EaA8ivG3y4ojj&P49g5&$mdU@H6|?R9MGhT>-xo>b;X9Lp_#;`09|cGOo@rI7 z8>Y*Bs!mac%*u}?9V@~}l>7PwKSZ?bbqv~$QXmrnu1C;{>I=lh!kAnF z1w@lPi0#B1=$^=#YePwOeX8GF6DF+Z#HV3@pPfkjXmWzymGtcd{on-xAOHd&00JNY z0w4eaAOHd&00JOz1cBCy^7_ui!>PB&c78T7dVX|dbad3;SMOGGrRD7M!rZQLJv+Bh zVoP%?3;C6~{KDe(d^Ve_l=8dt+4+UJTy`!yujLk3v-#ER;?B7(fT{SXzl~iQUSm0I8Q)y0LY*{j)^*_nri zyH9D?t}U;vt=+gW^NWu^+4%Z5j}nP->Zg+aHbFmlfdB}A00@8p2!H?xfB*=900@8p z2)twjE)DODo#gsMu>b!qk^b%_8z8z10w4eaAOHd&00JNY0w4eaAOHd&a3q1V!&778 zxq{&N|8ydqKC%chK>!3m00ck)1V8`;KmY_l00cl_kO;IaB|WjT^X}Bv*4F6g&GVz9 zC%K8fHov}HSe(B#mtVefeQx3U_59qee6BEe`}X4E`r^vsdg0cs+pJkuFliL`q}F%`Nie=`QY*Y`MfW(GM8KKFLHb>S^BcU z9RJa>S^81|{aoSiAAb_u|Nmnm{l`I4BgO&(AOHd&00JNY0w4eaAOHd&00JN&2n-KT zjN$wLf(DKt00JNY0w4eaAOHd&00JNY0wD165jg4h|NZCx>5u;51p*)d0w4eaAOHd& z00JNY0w4eaATYoL{QLjd{|~T&F*FbW0T2KI5C8!X009sH0T2KI5Fi5B|HBMG00ck) z1V8`;KmY_l00ck)1VCW$31I&}_&UZ2K>!3m00ck)1V8`;KmY_l00cn5|Nj5b{kx~8_f#>z&8u?6;`T{Fsc(|WPK`S<^a(*Kx9e@kz8 zfdB}A00@8p2!H?xfB*=900@8p2>cKTTpr$0PI6&m$p8KSC%Mdk{r?X^t>^;?fB*=9 z00@8p2!H?xfB*=900_hg`1}8pJhZ^~|0fajpZ{0V8wvWs3j{y_1V8`;KmY_l00ck) z1V8`;K;T3KTGy1Zot=p*H($SbbEW)ge%Z}mKk6a>libSRvLCV!jIF&wdE36vHt72X zdHS-!-29T3&93sFEBxL6o2}90WiB1V8`;KmY_l00ck)1V8`;K;We!;NSnp{{N*?89ECBAOHd& z00JNY0w4eaAOHd&00O5bFp>VB#H8}u#JTU%|9bAP$A5S3n~|;IzZu>e zKBN40=$pjv6JOImdR=)nsouPxBuuN!p1PmaOpobC)3f=xUUlqd!_{-a@wL{}D=GD( z6{U59XPf1xdV@KxZ5cJ)D3zG&>YlyNELZn`d4X=0gYN%%+vwec^Xr?1we5m-Z{v31 z=UVajt`@aN8(OheIr|q>{4+^}kKWSw> z^ib3;)6k2W$DVq#(HO#0%cwJvwP(0{tQ=@hq_L+B(=j~Lw)C>$k*NRfnRPZ>EIEum zm-XF4AuLi1PnN40uIJkVbs;MJ;pV-EYnzX?`-R6r?=rLNk2bcpH`nfMY-`2JJ~tj$ zZy37g>eZrl=h0^2-rbF`K%@w5vv8-dS=d-FY-xOGE~$wIo5}1>q}1!G(i(}Z)Gy@v zCTce=kL;44Pj;I~r(9qZJe4YMm#0R~CDj{hzj+hXljDbp=Qqx#R8>{Jddw|`C-S#* zaeLrtag9kWhDvknTBJ17)oEM@W{GiRMa^o~nPZk_i-y1QfFifam|~q+w=?jmZ#k=+*#Xsu&vF|A5Wc`Et;0cs-a~$ ztU~%IYCMfOJu)R>@30{Tf4mii>;hlZ!;mDlnV2n!f{{6mcbZd!#m$MIpY1lMPC4OY zsC0)pJs3->msG`!EH)^*&uGLXJ>IWw5{OsoCuDw^N^y((bv^V^R0XeRG;VP`?Kq3$ zO#}|aWnQzPO9^H}SWBI3P2aO!Pqr$aShaQAjfw3u$6~cuQpt9NW{bR3qh@#&+o{K- zI_j}q{eVVB4k{+~#Os;dG`-iD@lm*kuE*;B3^XbH(InYr zPOKlHK*O=i&5{?JZhVMge%-K|6{F-eosRO%IxrpEsxymxYM=mKu42~OU6FB@De%^e z-8NcZzq;!gUenJYSL`wto4c|=$+j$3BCjsZcsZX>e4Ph{zA_H`tZ5P}-DuFT4-8Ur z)h!4vzg%;-^JDyTij4`T<(lzMYo zXa>=kl@fW3D6sWf3l)3B-KJ|)nO-yN zCbb=v?qRy8FrzlH7qF%LNK(B!-EYYwHIZw7PVNq;)clmvQX;Do;Z-l&b;Gp8>6vb8 z5?P^SU`Ha6tSG3UnECQUC8bVHDZe6DM-D_th~rS)dPH*EcEtMixD{>KX6TQDGo3sL ztJ+Sq7^a0;nA32~x2jZ{QNnSf3F|rWX_$7}qyBDtV)PFQHSs~>Y;OEFW1o)Q z82$IuUyc54YHFmI{I4_rIXrphUxxlZ@gGA*;)C?(lm9*O50n2iF@5fzsqi0yy}EdB zz44Qj`Y5NgR-%dPdr-dNpy7we!{#Zvb!X0WPv~wTF)?_2^f%z$A|2u4*)OJ2YA&aI zd67@{Zu#yME$#ri1>&xtJ*a4|Njv2-b4!ld@VCWsb_gb#o&g{s5xN0s`Un0Y67OZZ zhEv)L{fOIbX5B{9xs4XK5iCIbtzTPlUE6AD&`0lv)_T6&DycbwnTzUJbtB_&!x#0NY&o89Z zwTnt?Dl(piT{BCEy3?eMksh9yWjXboZab2*+l9j2v{WIPR8(a#^ITC=>cxx7mxW+! zEeXmMz7@AFIVqvVUI2t|G#HY4Za+D(V_ENa~~%d|)1 zrD8iFinyL#vZIZ8!*;x=kbj~(hETwZ3+{kXd>&souZX&jp3`2F8$+&)&A)NU1-i#pwCC<@jA}&vnuMg+3+| zmzL~>GCJJPvvWV<3)AQOZDS!R(Sq8;Nc>8Ub3&)JkU;-r^K`-sG(!Nma@mdVY zgcFPRFMdU~jgGV>Uz8OkLrrF`y?)f9Mo`fMol!$?p8d;UQB!%XGhB47KZ1lex^>ni z8E7o@gYk>Hr@B$`?7j0mwtVp}S8h;Y@cwFCv0+=MP^*0Pbe3*P_!kUA{~(r={&kA~ zhl{5{-(SGBpZcCr?|UbIw);8v(@9ld>FZ30e2?2(bjhSwI(_d}S^AUR9O}_<+%{{;ygK>!3m z00ck)1V8`;KmY_l00cl_a0&R&|10V56ZC@@2!H?xfB*=900@8p2!H?xfB*=9z~B)W zR)!N3`2PRkDHbCF0T2KI5C8!X009sH0T2KI5CDNr0@(j|62UnLfB*=900@8p2!H?x zfB*=900;~o0et^|@N|k1fdB}A00@8p2!H?xfB*=900@9UCjspLJBi>N1V8`;KmY_l w00ck)1V8`;KmY^=j{u(kA3U96L?8eHAOHd&00JNY0w4eaAOHd&&`IEb0dP~Nf&c&j literal 0 HcmV?d00001 diff --git a/management/server/testdata/store_policy_migrate.json b/management/server/testdata/store_policy_migrate.json deleted file mode 100644 index 1b046e632..000000000 --- a/management/server/testdata/store_policy_migrate.json +++ /dev/null @@ -1,116 +0,0 @@ -{ - "Accounts": { - "bf1c8084-ba50-4ce7-9439-34653001fc3b": { - "Id": "bf1c8084-ba50-4ce7-9439-34653001fc3b", - "Domain": "test.com", - "DomainCategory": "private", - "IsDomainPrimaryAccount": true, - "SetupKeys": { - "A2C8E62B-38F5-4553-B31E-DD66C696CEBB": { - "Key": "A2C8E62B-38F5-4553-B31E-DD66C696CEBB", - "Name": "Default key", - "Type": "reusable", - "CreatedAt": "2021-08-19T20:46:20.005936822+02:00", - "ExpiresAt": "2321-09-18T20:46:20.005936822+02:00", - "Revoked": false, - "UsedTimes": 0 - } - }, - "Network": { - "Id": "af1c8024-ha40-4ce2-9418-34653101fc3c", - "Net": { - "IP": "100.64.0.0", - "Mask": "//8AAA==" - }, - "Dns": null - }, - "Peers": { - "cfefqs706sqkneg59g4g": { - "ID": "cfefqs706sqkneg59g4g", - "Key": "MI5mHfJhbggPfD3FqEIsXm8X5bSWeUI2LhO9MpEEtWA=", - "SetupKey": "", - "IP": "100.103.179.238", - "Meta": { - "Hostname": "Ubuntu-2204-jammy-amd64-base", - "GoOS": "linux", - "Kernel": "Linux", - "Core": "22.04", - "Platform": "x86_64", - "OS": "Ubuntu", - "WtVersion": "development", - "UIVersion": "" - }, - "Name": "crocodile", - "DNSLabel": "crocodile", - "Status": { - "LastSeen": "2023-02-13T12:37:12.635454796Z", - "Connected": true - }, - "UserID": "edafee4e-63fb-11ec-90d6-0242ac120003", - "SSHKey": "AAAAC3NzaC1lZDI1NTE5AAAAIJN1NM4bpB9K", - "SSHEnabled": false - }, - "cfeg6sf06sqkneg59g50": { - "ID": "cfeg6sf06sqkneg59g50", - "Key": "zMAOKUeIYIuun4n0xPR1b3IdYZPmsyjYmB2jWCuloC4=", - "SetupKey": "", - "IP": "100.103.26.180", - "Meta": { - "Hostname": "borg", - "GoOS": "linux", - "Kernel": "Linux", - "Core": "22.04", - "Platform": "x86_64", - "OS": "Ubuntu", - "WtVersion": "development", - "UIVersion": "" - }, - "Name": "dingo", - "DNSLabel": "dingo", - "Status": { - "LastSeen": "2023-02-21T09:37:42.565899199Z", - "Connected": false - }, - "UserID": "f4f6d672-63fb-11ec-90d6-0242ac120003", - "SSHKey": "AAAAC3NzaC1lZDI1NTE5AAAAILHW", - "SSHEnabled": true - } - }, - "Groups": { - "cfefqs706sqkneg59g3g": { - "ID": "cfefqs706sqkneg59g3g", - "Name": "All", - "Peers": [ - "cfefqs706sqkneg59g4g", - "cfeg6sf06sqkneg59g50" - ] - } - }, - "Rules": { - "cfefqs706sqkneg59g40": { - "ID": "cfefqs706sqkneg59g40", - "Name": "Default", - "Description": "This is a default rule that allows connections between all the resources", - "Disabled": false, - "Source": [ - "cfefqs706sqkneg59g3g" - ], - "Destination": [ - "cfefqs706sqkneg59g3g" - ], - "Flow": 0 - } - }, - "Users": { - "edafee4e-63fb-11ec-90d6-0242ac120003": { - "Id": "edafee4e-63fb-11ec-90d6-0242ac120003", - "Role": "admin" - }, - "f4f6d672-63fb-11ec-90d6-0242ac120003": { - "Id": "f4f6d672-63fb-11ec-90d6-0242ac120003", - "Role": "user" - } - } - } - } -} diff --git a/management/server/testdata/store_policy_migrate.sqlite b/management/server/testdata/store_policy_migrate.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..0c1a491a68d58e019b5256b60338ab8b5f5e40d4 GIT binary patch literal 163840 zcmeI5O>7%SmdDwmB~r2}#_=Q_CDDXd;@CtanawYY9vEI)q7#j6*^w+b9wTTr$s*Mj z*-dvhWov>gKsyO$2UyGwHpc~KPji^V%wmAqoaeOI!(jKaK(343!(8^3!vdMZzW!u4 zKdcW=7MABv%Otz%Rdvzx=C()cwIJFqONK!$w*{EFNHWR^jC>QC=_~={-2}& z?&EcObId)USI+l-)Y~^hi`qYqo5JyLO_^8X-%kDZ#J#DD6XR3y@vp`=!dv6#!pAsi z?5oi4LtoNAdc61Y1b=yk3&~22J<=aEsez)@F;weVY-nnyt&4*Fda677QiT7oz;!QK z*>dfX*k+opDpFIFs#T`zqM>dxMHf3d(?q#8Jon`CdZ~E3lvrI`DSebE?_EtfadRzE z9$s1bT-iBUIr(^FI>HwU+-F5Ysx;Z~0>e||PMdBQuUs#UJXhizwYHov*dt@!JFRo5 zNG;}OZAtnTtJyHzH1?=1Ymym&7*Y75-mG0WDOU^IfT-&(4UR+(fohaA0EnVnhTM`XjY?KpMZ?2bC-(PbI zcY@@JJsinR0<>Zq~$0Iz?b6?!G%)k<{o=yanY?lq_)K`t8R!wbs zYL#`78oMJ`nFY0+P&zH9$<_I?VrrE&nRP;6x?f!MTZlQvoZ;o_ywX&w+oo>4S}lDy z)rPG26z*9|&^BaU@0f*6+f1yKt`;}1-%iX)ZFz3KEGq_UIJT#;IzcWctTd(#$@B>u zyKT4gd)t}A&K~?jOwGqO8L&3Dplfi+6BGO;ew4vkS`!89C03d0H>WlWnv->We9)Zw z_rTfh3`s4(Emgi*8V(p^ajNp*;)Tp{iAXurHko-v`Vj1VpT8@vRDy({CEV~MPGv4b`7gVW+`r|y5ToKE3u)9svZ#AW}3p9fuyRcxkga7O0}Dk zVFs~)R9`)+F78mz$lC>^9vVZFJMz$EyD>wv6lOeBwQW(V)ijDbEZ7TuS2tM89IP^h zCK^u_rUlyJ2(&e|)~OnSk!f|&$!|$Yr!G~Ej@DP6DLb;JDlMjvgSHi5m8;9mUQ_(I zYZMflQl*F1)UT);lF>0U$gk^+1%|m_psFegt5S&YIR}5fOa5loOV-5JjZdBfcwI(vzcL8sKq^aWenLG`lwT;G#-k45;?r@1e9%USj0dhf=A zZmXAZ0N>>wD87(0BGTDrPW=wdjXG^}N>uAh)z}VANd0C@rcM(xns$-L@M7SwiB)+> zi0;no#lQ?u8tgtE8)J51+B+T4%&`88QyDp z?3~l_$M%s(9)vYi&6`=%9L~vUYjR5>x8a=B=MLTABG<^j%XPWu{M{USb^X4ZL>|6B z$&NohOEIGWDZr4G1~o(0EuShJWvq;#H z*+X$~9_mRu>p_YiZK*NJ>wD7oDcUg6 za|7IvpykzPi@OtJatRa=W%3}Z7Cgb=Ku(??o8XscM~yYtghj!97dF?~q0slHWAx0$ zS3>lM2MB-w2!H?xfB*=900@8p2!H?xfWT7-bPL=oo3R_SZ%l69iA7IF$D`4xxvs9% zh3ZmjDZ5aSa;b%Em0egUWHW_@Og5j(q*6k?nyI8y>Fk1#T1cf6LjGbZdoi6^%w?9c zg797{eKD0Xm~Jdq)fR0G?a;1~WK~H;wxq08WtyrKmh378tIFypbE~)JF3t(5)M7rn zm|9HD&ClJC^leHze}1W0EMB@a_vyWl*S`Gu%}{8H+R4Sg4$&VTAOHd&00JNY0w4ea zAOHd&00JNY0!NI%+3@D%e)b>i_5U}a_%}zagJ>=YfB*=900@8p2!H?xfB*=900@A< zQwbao&rZ7c71;Ox$3yY>Q;Q%I1V8`;KmY_l00ck)1V8`;KmY_@Bm!N9i^n!M-=5vr z*oa0ipNvNLa}m9mUS2BY(^nQUOILFX*<3EOa3v#@7FJgB`Q>~ezg)U9AFQ(~c{Zy`y$uFhT_U-@aj44uB5SB)Z>|INW zK5VdH{UuT<`cMJ=y=Oh&``Etz|EEy=PcM=h(H9T^0T2KI5C8!X009sH0T2KI5C8#} zKsX$m#OMEA8h8Z(5C8!X009sH0T2KI5C8!X0D+@NV85ULH}C%+JMp{F^i_Jo0|Y<- z1V8`;KmY_l00ck)1fDknPo~Gt<~CVu;T8Hx<(Zk-+p$k88(Hl3S zCu6Z_wayxOy`IYJ54IK7$Q2s7)PC;h{_&0C&1;)%_3mn?qhyuTqg(4jC9_()`=eVe zefR#|)|K@AJIkG>x}3daebc&-$}Fbyi^5WBuA*vBUs`(NbQ_XE!VK zo1mQ<+hI+$ZT__0Cv(e9wNs-nW~Z-czvcXbt$FlmjedT`2XE^wAY z&X@WW^441$lcbQ%s81pF2l@q{Lb8qhDCEX!uJu9vhg+3K<5qnobM--KRsX29^ii&| zaffZLrmt_^EZk_9O2(aHPa(pE!eTnJG-rcfNT*ZTh5J&gwYwm-YI*Z}`E~ZJnyP9l z{l+W(#IE;Gf%e+$L`NKbfx00@8p2!H?xfB*=900@8p2!H?x90dZeg>&4nJ+Co7oSUSK zfxQ=d{r`(l{NLlhI0_v^8$kdBKmY_l00ck)1V8`;KmY_l-~}X*3N!8?_o9rQ53|XG z+$&+P|9=yTe;xnL3&;;$0s#;J0T2KI5C8!X009sH0T2Lzqe$Rfc$3@D$0f!N^!fkk ze+|WJ^n?cpfB*=900@8p2!H?xfB*=900=zq1iG({O>b_#5{<@AMlatu6^-rZTM_oN zl2s(_3CO^E*R009sH0T2KI5C8!X009sH0T2KI5csYMbhv14^VF&BXslXi^#}Te zR9=6ut*}O}(8x6Q^J@UbW|Mww;NIL|`D|lu-b`-f^}4UxTx#xP^ZGv*|6_>$@Bjf2 z009sH0T2KI5C8!X009sH0T4Li1o&`na&TiH(}>0YH#E)tI&|Wj_`jU^yQ$xvxHoli zVtgt-{?+(K`0vA8;bYvd$G!^vKJ+F1qsMzMPwMwB(|BRtBTYVrD~Pwx@f4|OwmR2M;1l7W<&mN&oR6O zCzsbt#oMLC>e@=_qeOY{rj`>o*AnI7m6gwxoeq_gk2j_xe4)U7Ry3qalMOF0JT>mL z>2~qT^^$e2KWk+}^iWQmqpp_|276@8dwp;Y6{*GCtSw33Vl`WPZW?>kmNm(cRYj~x z28o)_hTLNF<*LT$eNC+Fy25UXWccNpl5Uu`Kuz!p-&$Y2QCz>9xK_Grw=Oxqd~=v|d_UE^Q>NuIZ#E>TEJu ziADGv&vnN=D>V!0rirQ@#UQ)1j;9Arq+iZ93M-W=@0F*MoS5J*@uS9#U5{YB#GYI_ z9^rYO`{J%;F;=4WbRuXER$5SFzB4*X)6}M?G+7s^UpsP@S;)!>rPE@XT%9i~rp{QC zSts>rpjCNO}m8M$VHZ|wfYH72nHe|)8PR|yCnwNFGV-_|oFR@a(THL&T zJ26N5nRD}HSut3{u`G?%Ngw5emBzFonLc4-x9xU*Z!2=x*}iCqA^F%Q1GeNA^vr3h z-<;YkXinDg@j-Lymvem#mF_dA>ys1wS)P+Ui?xexGwLx(PmQXZ1cH^C3CW*CB9_Jd ztmXJ9s=}%#8nieoZ7++nPGLI`of)0B=u42>uC=tt*2FDUH~dy*B{o!1)dOPNOjB4h zkW^JQSF>fSRJ$n|byaHxr26Vnb#aG!Mh+?<_0Sla+>wVS+l?8Tr7+{6s%?u>t)@|I zW5HhNySl+z<_MH2{Lvt(FfGsyN1&~#wNBLt3^%KbPJT;LI(4aPbhN(mOxck&RcSGW ze5$Pgt6W`f_L}0yU8BI;lqx;6rhY};kc^I*L9SS5EHHNc0##K}Se3lGZ^Zlat${CE zL1Cnf#vXKJVkJs#>h_LAO7^^m>E5iN?%0yP<+miA4w+&`RR8&X=d=US5D~Lq`*cdI zy~W5w2AV})&aT?&zZvd&X6?(R$f50uRkh|BYA;hZb_4qLavDsvG6FvC0KUr_p@Tti z-@YpueOj2ZYZt7S$(@M^fB7`mo$-8-H{AW9vzHhdbV|KUU$C_uR4=Q~^*wp_ScE@) zn)`ycoK;V*_ijAswt5){@Lm3a;)^3A!k2C4)bGG_rqecOF}1!_jqT8+*Kf9D>NJ<4 znG$&nFR%@p3zdh&-Ht9bnAntCGM(EiJ;ZbmU`9P+&tOZL@d^I@)1#(5zEcRj_Y;+H zgwM=!UCy&AH@u29wI#`ldw6Wnn%pc$(zYWvk*vtBpq%{t78l`XXStt}t0M>EO1Q6M zLF;jogSO+&U-z2PI_Gr!v3(?x2Vo6W^CrVIF>`X-n%t7eZ8#_OxkER&$Tjlsa$W8@ ze>aC-UBB-pk%#Y3vg41>Qp_ko3NU1)LCuhL%cqJIWyq||SkgDm^F%FQzYmA|^a8$y zc!C;hbefXp%#3TWXB*`kwTCiuO*4!PqcNs@jy~R=`;Yq@-q3 zeK-mc&1E%-mb?_m+yFNuXnFP7;_hshTml6|nLLQ91y3+Ikdx=fCivyqQDeIVOcZ zh<`HuKe1m;|5NPriQiG-KiF$^_qqG(TM_=Iz;z4Wz%@Onwcw!cJII~WQ*;~5oEsd_ zgF^1Wu;Qb+03Q_T3m1<+osIB{H@4V{Or@$fsy3;+ypju^!W% zxD{HiC1^UX>$m4D-h7}X=?^+Il{IIh-V~db2ekb6x!^&w3RX@%G&k*N>TA%(!d_l! zq=DJ+o?&Y7-WyNbg>u68uivs-ILuF)FJDiv4>QN%UWs_HGw+p1=Gh0Fs{laz*6&)yEm z>YGd|u@uw4k=N^a(BZhHh`YI)XKu@`l&+U(FL$}Pv0Pj!B`>@-!CyNw$^|*~+QuOG zkKRyTjqpF9$>_?imY7xo6;q7>QZQ zUQXzs7Gmq4Y+N@t8N})qt8TlB-CK(e8E0Tw?F;V6_Rx{G{F}0FNk@~(^RGN@QsYt) zZJp7HUOWER_N1nMsy|!|%s=deR|j=AAZcsN@q@ul-2+{yc>K;uE4F<0wxwLVLi_pU zpkkeK9YC%A-P3uxC}D0GI{v|(PMZ4^{|}p|cH5u9wIBGJ(eP_0-`Vw?Yw-zQER1v} zM1IKSE!r{}mQKIB#a;NbT^t&6;-G8j+fnmJj-z!w=l+=pfBrnz4ck-t{%u!x3gqp> zo(qnzfqWB_HSg+ePfIi{GB+NR-HCT1{QF6+oAd<%zhex90)NKYLFW>5slJkKSn!u8 zH|TejkHd2jKAGg6I2RHF-Z@zDnP7i|nL(TI8^Mt2gRcROKRJ9f2=uM*DxL2&o(JlPu>`V_Fyd7*yAkWpNH_`X!(n7-AL+H2@#KGX5wD14_Iu!rf zUCHAW1V8`;KmY_l00ck)1V8`;KmY_l;HVHd8{VAU&zA<6-~Wf-|2ryLMLR(N1V8`; zKmY_l00ck)1V8`;K;Uo$@cVy<qOK>!3m00ck)1V8`;KmY_l00cnba0IaaKO8rd z0s#;J0T2KI5C8!X009sH0T2LzqeQ^G|DTJ08=^lvKmY_l00ck)1V8`;KmY_l00ck) z1YSG>VJ;ks;q(75o?_7>5C8!X009sH0T2KI5C8!X009u_CxG>TKM@>*00@8p2!H?x zfB*=900@8p2!OzgM*yGyfAMsR9)SP|fB*=900@8p2!H?xfB*=9KtBPj|NDvH7z987 z1V8`;KmY_l00ck)1V8`;UOWQ0|Nq6)DS89~AOHd&00JNY0w4eaAOHd&00R94{ttLC B#c2Ql literal 0 HcmV?d00001 diff --git a/management/server/testdata/store_with_expired_peers.json b/management/server/testdata/store_with_expired_peers.json deleted file mode 100644 index 44c225682..000000000 --- a/management/server/testdata/store_with_expired_peers.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "Accounts": { - "bf1c8084-ba50-4ce7-9439-34653001fc3b": { - "Id": "bf1c8084-ba50-4ce7-9439-34653001fc3b", - "Domain": "test.com", - "DomainCategory": "private", - "IsDomainPrimaryAccount": true, - "Settings": { - "PeerLoginExpirationEnabled": true, - "PeerLoginExpiration": 3600000000000 - }, - "SetupKeys": { - "A2C8E62B-38F5-4553-B31E-DD66C696CEBB": { - "Key": "A2C8E62B-38F5-4553-B31E-DD66C696CEBB", - "Name": "Default key", - "Type": "reusable", - "CreatedAt": "2021-08-19T20:46:20.005936822+02:00", - "ExpiresAt": "2321-09-18T20:46:20.005936822+02:00", - "Revoked": false, - "UsedTimes": 0 - - } - }, - "Network": { - "Id": "af1c8024-ha40-4ce2-9418-34653101fc3c", - "Net": { - "IP": "100.64.0.0", - "Mask": "//8AAA==" - }, - "Dns": null - }, - "Peers": { - "cfvprsrlo1hqoo49ohog": { - "ID": "cfvprsrlo1hqoo49ohog", - "Key": "5rvhvriKJZ3S9oxYToVj5TzDM9u9y8cxg7htIMWlYAg=", - "SetupKey": "72546A29-6BC8-4311-BCFC-9CDBF33F1A48", - "IP": "100.64.114.31", - "Meta": { - "Hostname": "f2a34f6a4731", - "GoOS": "linux", - "Kernel": "Linux", - "Core": "11", - "Platform": "unknown", - "OS": "Debian GNU/Linux", - "WtVersion": "0.12.0", - "UIVersion": "" - }, - "Name": "f2a34f6a4731", - "DNSLabel": "f2a34f6a4731", - "Status": { - "LastSeen": "2023-03-02T09:21:02.189035775+01:00", - "Connected": false, - "LoginExpired": false - }, - "UserID": "", - "SSHKey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILzUUSYG/LGnV8zarb2SGN+tib/PZ+M7cL4WtTzUrTpk", - "SSHEnabled": false, - "LoginExpirationEnabled": true, - "LastLogin": "2023-03-01T19:48:19.817799698+01:00" - }, - "cg05lnblo1hkg2j514p0": { - "ID": "cg05lnblo1hkg2j514p0", - "Key": "RlSy2vzoG2HyMBTUImXOiVhCBiiBa5qD5xzMxkiFDW4=", - "SetupKey": "", - "IP": "100.64.39.54", - "Meta": { - "Hostname": "expiredhost", - "GoOS": "linux", - "Kernel": "Linux", - "Core": "22.04", - "Platform": "x86_64", - "OS": "Ubuntu", - "WtVersion": "development", - "UIVersion": "" - }, - "Name": "expiredhost", - "DNSLabel": "expiredhost", - "Status": { - "LastSeen": "2023-03-02T09:19:57.276717255+01:00", - "Connected": false, - "LoginExpired": true - }, - "UserID": "edafee4e-63fb-11ec-90d6-0242ac120003", - "SSHKey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMbK5ZXJsGOOWoBT4OmkPtgdPZe2Q7bDuS/zjn2CZxhK", - "SSHEnabled": false, - "LoginExpirationEnabled": true, - "LastLogin": "2023-03-02T09:14:21.791679181+01:00" - }, - "cg3161rlo1hs9cq94gdg": { - "ID": "cg3161rlo1hs9cq94gdg", - "Key": "mVABSKj28gv+JRsf7e0NEGKgSOGTfU/nPB2cpuG56HU=", - "SetupKey": "", - "IP": "100.64.117.96", - "Meta": { - "Hostname": "testhost", - "GoOS": "linux", - "Kernel": "Linux", - "Core": "22.04", - "Platform": "x86_64", - "OS": "Ubuntu", - "WtVersion": "development", - "UIVersion": "" - }, - "Name": "testhost", - "DNSLabel": "testhost", - "Status": { - "LastSeen": "2023-03-06T18:21:27.252010027+01:00", - "Connected": false, - "LoginExpired": false - }, - "UserID": "edafee4e-63fb-11ec-90d6-0242ac120003", - "SSHKey": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINWvvUkFFcrj48CWTkNUb/do/n52i1L5dH4DhGu+4ZuM", - "SSHEnabled": false, - "LoginExpirationEnabled": false, - "LastLogin": "2023-03-07T09:02:47.442857106+01:00" - } - }, - "Users": { - "edafee4e-63fb-11ec-90d6-0242ac120003": { - "Id": "edafee4e-63fb-11ec-90d6-0242ac120003", - "Role": "admin" - }, - "f4f6d672-63fb-11ec-90d6-0242ac120003": { - "Id": "f4f6d672-63fb-11ec-90d6-0242ac120003", - "Role": "user" - } - } - } - } -} \ No newline at end of file diff --git a/management/server/testdata/store_with_expired_peers.sqlite b/management/server/testdata/store_with_expired_peers.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..ed1133211d28b5b2faf4963e833b2ac521ff7fe0 GIT binary patch literal 163840 zcmeI5Pi!06eaAVXB}%esuCv+9D!WQ(InJ7}C2{yi6df2_N}{xi^~b9}>$Ml@a7aE; zW05oT%utrRNCB?1Mi8J#i{@CMIrY#(QUqwW*PMzTg7#7%*P=c2(%T*iY!3a+U(WE4 zCDoh4t9)sh!+G!b-n{qu{oe2Q-pp{g{r;M4u=t*;H6$aR4!suQxX_=(nl&vQR27*e^;`WNV*8gtsTQ&?Cl4m?);2DP>nH`tMJ%R8*I zt4IyzX6;M*KC9X=+%$I7lr_naRYj~y28o*QhTLGcN)?UK=c-sfc7@#($?(h7CEYO5 zQWLzw_qJBo3tJE3cZ(0~)+KH&ZfX^5#}?b!EdT z;IWHu6_<-!#f`<{cHHWkPGqUGiNxL52ruRbnMi+}qR>D6wQ@AVXL+tY>{*c60^Kwp zwWSzjGuHm)UK8n-vyH+^rIvQemlDtJT#WEM&pm%&nSmu@y3(t1Zz1LwbB33z^KxCS9GJTGYPIxT zS8KB3Q@Cd>LEDgZy=4|QZ8N@9TrTXc?Zl^~raX14Br681Iku;Fnv=DEvDcis z_rMt{&ZES>phv#I^V(p>_jNp*;_@KH;AXurHkoa*VVj1U88;*~nDy(`&gT`s4 zl}<2D>kzg#(wWg}ioOK7=^9pp%u?J}b;ECfR$@&RRXrefz%+%`14$KCbB&;6m1@={ z!wg~psjhldT|A_ok+%y-Ju&(wx8%Obc4PWxDa?4HY6qfJt!fl^Sg;rRv2L)2Iap;1 zO*Ed$ObfKb5ol^^wN)_!Bh%`lli!e()}B-`T3T0mrX0$esx+8F4%$|LRc=qNcbekI zU8SH{m&zTqrhY};kc^g@L4I9lEHKRd0u@zJScO7_&pG(>EgvCT5oVx_#vZq1VkJsV z>h_^TO7-3t-^>Q(MJZ#k=uT<6np z&~0@x&fvTJ1H~6|21GjB%&FgjxlyZWPKl~rsTw<^38~*~$<%3LM$<0x7+wtQH?b=9 z3DGTGsxh%HH)J}tSGtesp23Vd#9qRd(!(SC%GE(3({~7=^LeBkj_~P8uFZK?dpF4Dei(DoDF7L@5=kMmw*7f^t5_$OU zBs>22EX9lhqyR%!YSau_w|uHdQHIRQj3r$&LQmB4^`~&SFD~F~h$pDATB|N;&LZJZ zW>3W4d8jAttOqH6G^N@gukT4eqG<1w=#35iq$+huZUmflKuW6D)hB}x(X>~SXgNxO z%nfjTf|ggGE$&W?$t6%gl*xmrTJQk91357}G{P@V4jOB&35$aJDQvE@L!qx_*oo-x}M!A053iHat2yYObrx zdqQPCIiHy>OWEXfrowX5`Aj-LozBc<)5)Z;S4o#s$y8=qNKPkHaUpj*nY^7&&16%# zY-;{yGDZ6a(~X&m+Mp9dhji9RvZ|ya`%=cLGDTGi^LCYjRb}PlsnvT^x2J?;a%L_w zlblIT-I`jL^aDzpot-Zf3U}^I{qUn7gqM^5`18;>wUdkeE<}HLfdB}A00@8p2!H?x zfB*=900@8p2z&^z=enC{8ae&CM;&<>wZQ3kyqZPiobT zI6VzXWAw}pdhi6@`!7r<=ck2yJe9ninY*2$oAr~~e0pv^m9lUDPo+(f{IoDXSmfkd zlJsSRY3nae-!QQLZd&gj{lLEd|NBtv_vcBC=nDvd00@8p2!H?xfB*=900@8p2!Mb~ zARLa4;rstC4QxRG1V8`;KmY_l00ck)1V8`;K;WxK;I!ZWH}C%skL#h!-@a6&4PGDs z0w4eaAOHd&00JQJWfOR|8NQL-Wzn0H^n;m-a@Lsfw;qJlmaz%TTnP0rWbFi^no~^30N;V}6 zYuW01nWg=e*3Hbr*1FfMTs)b-O^+PU`ofXfj(@=;z#o{exQSQC7$_lc&+>R(<<8b@)_WNxgTxzOb{q z+W2r&{$PJ`L6#S!?Bk{E(bM^Gre#X;iIzN-mOgX|g$=hdSJJL(6IvO7^gN?JL!y|QuBkju089^PEfRn{{1jh&~v+D`MpJERaNqsz>b>CMaw zxm-R!m#1G3Fg*&1^!s%I-_SL$|A%A04#oZ}_Aj)-3j{y_1V8`;KmY_l00ck)1V8`; zK;V@j@Mbv6^*c8jwljK9u?_Lz>{zAG85(>2uY_X%8vC1=^2&4&4FLfV009sH0T2KI z5C8!X009sHfv+%uWSDVhxi=|n7C6hTFGI6oHg=XfD(v;E15 zc_B}a&oCd4AvNXpn;f<6jZWM4=!pc?xm@Z+Px?4b+gdu)Uao28_5Y#RFGF-zs`&N6uGf4cwwkAxw*2?8Jh z0w4eaAOHd&00JNY0w4eauR4K0$o>DJ*IxB*qkSL%0w4eaAOHd&00JNY0w4eaAOHd{ zL%_WLkM;k{5JXuJ009sH0T2KI5C8!X009sH0T6ig31I#I>gxzC1OX5L0T2KI5C8!X z009sH0T2Lzmn9H2E5QB#FN+c7K>!3m00ck)1V8`;KmY_l00cnbl_r4e|F5)q&^QnP z0T2KI5C8!X009sH0T2KI5cpCEL}ULGy3G9|bm{ZhKVABZ@n2o~X#Dob@OW(av*GRV zUxoL>7r0*xeHQvn=x6kgUN^rs!oNGgg=D45j`YWM*d$rY%)z(6{#*tl?u~!(NGVVqKoDa8H#e%?$RIj82wvtWpS%m*eS+W zHx% zETo$zspO`pvHVhbd;v4bx&!sE>gb^b@D zgVh|%(%2s9qZGH&nARuLCu|%y-OitEMGiaL7xghDAKOI0mfV7#IgNLlQOX-*bYQzMyo0M66B_9Ee*0YabML9zg1a@HC0sgfY<@k6jl!;RaDK@Y{@Fs ztV_n8sx<;qUG=ECct|}X2NjTdV)RXJ$$gXU#`MinnDIo_4n(P1)hM>HU@!Dz-CzxK z1j-ctXpoed7HEef(A3mwt6~I(o7F`pzac5DJ*i@}w65|@Ig~Y3X)uL+s;vO4+@4(T zG{uj*N`bd7l{;un{ffFF87(t|T(QnrVC?z@DypKe3VC(ki1+7P17Ea)!ax~~J#NXw zN|c(^?L&!_?0FB6BP=pOJ?QG>g2P zUA5DHGu(B|+Lud_Lpv5LYSlB;PNr-e2lVUZ)R<~z1bo~Xe3vysdxPMqeOF@iMPbUW zU9dJ|ha(aG-K$)C!t+7iaQBDKPGY3jDRnY^!Pa(Aovbd`_w2hDBK*~>+;iS?Rvo#{ zr{SR6>SUb3clig3FOCcdU$&W3zXQ{mR@0otRJ&3&c1V+6zuA(h(_D&XO5`!Tz}9ar zRO%CVTe?(ZVqI>?bZoD5AJaX98Fh%gge|3qNBEVigQh&bLkOMEBjs>}Pfv1f&a)~v zyoyz|A<2rnd!g5w+$=}Zwj(!@tjMmQl=$=>7vU!-xu1}$BM0J2xZ9zi^|;AF+i~Zw zC(US`V>H;1;a-*=P9 z!*?gy@yBN=W)vU=7_w5MX2`nbQ$>n0WL9P@>6+$wqL!~eg~NSu0bfHrL5d#6NiZ0ILdsY`Mr;H(2uQoXJ|8H9-DvYJFo zUJ7Jxfa?>qy!vc$ceYC|fdZmT9z@lG2k0HhiP@nMesOZpSaVHS6x>f?T4|4(tL^CM zZ$o_Oz0gHr{4d8Ijolgjx5%H5{&i$>xHR&g7ycuB`NFS<{xt+~JnlbV(rJvVN5>DlyIo2=V5})x75NlRMSQc~KGtKp6SqRkwFFJa zb^Z37#hVW_B>i!Vrn2U2)SF_{@_?5AJ{R0;R>8`thvrE;n)(`aV&NpOG|<3ocrP)v zxcSzL&O$oX3-~TQESn{NOX4Osz3GZUOVzt~vCvmR^n0($VLOs0c-V zD{OksMORxI>DjHdELUlbW|a!8gec+~YDM)H=1o;Iyh7%Vs7XrAtLJ0~Wc5uZl~{`D zK9SeydC=jwrHH$EHqSgQyHs2&(z)El!uDceshG&UIl|w)KF9?*_1eZD`1jsYz7gTS zN0ZSjLCZ0l*ct2Y@h1kDOicem=SMb!%e!?4V;TB;_7~?_(rpB_}zdvs#F) zf3k7iJjozd_F3h?RqV-HbjUaZ%W7ZnjBE!TY0H07)-CC1GBNwcizYQL717oi9q7%A ze`Zf=_FnG}7d`V2JK-C>I_r_NHRkxi;7Q#xU8wlvJ6EjO@}p~(a_tK3_pb#N>m2J0 zYW1Hzy+s!#%oB!=e{iRh=6Q<$hm)sv+h4-9pZS_m|7$0IuuOzs3!WRVmjxi7l{2A}`I+vhJ^_6tPg1-26AA8_b0IO{or4u$5B4{h8MFz%5%ifp_!{8&lmB}(2=uM%DxL2*FOCV{yEeja zCI$|_02DJa_z(Ks_Qk6f!uI=?;n?eKcljOM`%VyHFw;bM(yHF|y91Z;iS|%D!oTwl z*Iu+*BP(<|s!o?N$W!Zqoom6i1oB*MdJ}zrE-fU?a|kVWg4i3p6S)81J%fQQ2!H?x zfB*=900@8p2!H?xfB*=bUjlgk-}%)odIka@00JNY0w4eaAOHd&00JNY0xkhu|944X z3j!bj0w4eaAOHd&00JNY0w4ea=a&H1|L0e?=otur00@8p2!H?xfB*=900@8p2)G1r z|G!HDTMz&N5C8!X009sH0T2KI5C8!XIKKpN{r~*x7Ci$25C8!X009sH0T2KI5C8!X z00EZ(*8eUEY(W47KmY_l00ck)1V8`;KmY_l;QSK6{r~4zx9AxNfB*=900@8p2!H?x zfB*=900_7QaQ)vUfh`Dt00@8p2!H?xfB*=900@8p2%KL6SpT12-J)k800JNY0w4ea zAOHd&00JNY0wCZL!2SO&32Z?C1V8`;KmY_l00ck)1V8`;K;Zlm!1e$0t6TI81V8`; zKmY_l00ck)1V8`;KmY_>0$BgMB(Mbm5C8!X009sH0T2KI5C8!X0D<#Mz`XyTi+vHI zKfFKy1V8`;KmY_l00ck)1V8`;KmY{JAAv9z4n^_(|MRC<^aun%00ck)1V8`;KmY_l z00ck)1iA@e{ohRl`yc=UAOHd&00JNY0w4eaAOHd&aQ+D3`~T-pr|1y~fB*=900@8p z2!H?xfB*=900?vw!1}+N2=+k$1V8`;KmY_l00ck)1V8`;K;Zll!2SQ{Pp9Y+2!H?x WfB*=900@8p2!H?xfB*<|6Zl`7K~6CM literal 0 HcmV?d00001 diff --git a/management/server/testdata/storev1.json b/management/server/testdata/storev1.json deleted file mode 100644 index 674b2b87a..000000000 --- a/management/server/testdata/storev1.json +++ /dev/null @@ -1,154 +0,0 @@ -{ - "Accounts": { - "auth0|61bf82ddeab084006aa1bccd": { - "Id": "auth0|61bf82ddeab084006aa1bccd", - "SetupKeys": { - "1B2B50B0-B3E8-4B0C-A426-525EDB8481BD": { - "Id": "831727121", - "Key": "1B2B50B0-B3E8-4B0C-A426-525EDB8481BD", - "Name": "One-off key", - "Type": "one-off", - "CreatedAt": "2021-12-24T16:09:45.926075752+01:00", - "ExpiresAt": "2022-01-23T16:09:45.926075752+01:00", - "Revoked": false, - "UsedTimes": 1, - "LastUsed": "2021-12-24T16:12:45.763424077+01:00" - }, - "EB51E9EB-A11F-4F6E-8E49-C982891B405A": { - "Id": "1769568301", - "Key": "EB51E9EB-A11F-4F6E-8E49-C982891B405A", - "Name": "Default key", - "Type": "reusable", - "CreatedAt": "2021-12-24T16:09:45.926073628+01:00", - "ExpiresAt": "2022-01-23T16:09:45.926073628+01:00", - "Revoked": false, - "UsedTimes": 1, - "LastUsed": "2021-12-24T16:13:06.236748538+01:00" - } - }, - "Network": { - "Id": "a443c07a-5765-4a78-97fc-390d9c1d0e49", - "Net": { - "IP": "100.64.0.0", - "Mask": "/8AAAA==" - }, - "Dns": "" - }, - "Peers": { - "oMNaI8qWi0CyclSuwGR++SurxJyM3pQEiPEHwX8IREo=": { - "Key": "oMNaI8qWi0CyclSuwGR++SurxJyM3pQEiPEHwX8IREo=", - "SetupKey": "EB51E9EB-A11F-4F6E-8E49-C982891B405A", - "IP": "100.64.0.2", - "Meta": { - "Hostname": "braginini", - "GoOS": "linux", - "Kernel": "Linux", - "Core": "21.04", - "Platform": "x86_64", - "OS": "Ubuntu", - "WtVersion": "" - }, - "Name": "braginini", - "Status": { - "LastSeen": "2021-12-24T16:13:11.244342541+01:00", - "Connected": false - } - }, - "xlx9/9D8+ibnRiIIB8nHGMxGOzxV17r8ShPHgi4aYSM=": { - "Key": "xlx9/9D8+ibnRiIIB8nHGMxGOzxV17r8ShPHgi4aYSM=", - "SetupKey": "1B2B50B0-B3E8-4B0C-A426-525EDB8481BD", - "IP": "100.64.0.1", - "Meta": { - "Hostname": "braginini", - "GoOS": "linux", - "Kernel": "Linux", - "Core": "21.04", - "Platform": "x86_64", - "OS": "Ubuntu", - "WtVersion": "" - }, - "Name": "braginini", - "Status": { - "LastSeen": "2021-12-24T16:12:49.089339333+01:00", - "Connected": false - } - } - } - }, - "google-oauth2|103201118415301331038": { - "Id": "google-oauth2|103201118415301331038", - "SetupKeys": { - "5AFB60DB-61F2-4251-8E11-494847EE88E9": { - "Id": "2485964613", - "Key": "5AFB60DB-61F2-4251-8E11-494847EE88E9", - "Name": "Default key", - "Type": "reusable", - "CreatedAt": "2021-12-24T16:10:02.238476+01:00", - "ExpiresAt": "2022-01-23T16:10:02.238476+01:00", - "Revoked": false, - "UsedTimes": 1, - "LastUsed": "2021-12-24T16:12:05.994307717+01:00" - }, - "A72E4DC2-00DE-4542-8A24-62945438104E": { - "Id": "3504804807", - "Key": "A72E4DC2-00DE-4542-8A24-62945438104E", - "Name": "One-off key", - "Type": "one-off", - "CreatedAt": "2021-12-24T16:10:02.238478209+01:00", - "ExpiresAt": "2022-01-23T16:10:02.238478209+01:00", - "Revoked": false, - "UsedTimes": 1, - "LastUsed": "2021-12-24T16:11:27.015741738+01:00" - } - }, - "Network": { - "Id": "b6d0b152-364e-40c1-a8a1-fa7bcac2267f", - "Net": { - "IP": "100.64.0.0", - "Mask": "/8AAAA==" - }, - "Dns": "" - }, - "Peers": { - "6kjbmVq1hmucVzvBXo5OucY5OYv+jSsB1jUTLq291Dw=": { - "Key": "6kjbmVq1hmucVzvBXo5OucY5OYv+jSsB1jUTLq291Dw=", - "SetupKey": "5AFB60DB-61F2-4251-8E11-494847EE88E9", - "IP": "100.64.0.2", - "Meta": { - "Hostname": "braginini", - "GoOS": "linux", - "Kernel": "Linux", - "Core": "21.04", - "Platform": "x86_64", - "OS": "Ubuntu", - "WtVersion": "" - }, - "Name": "braginini", - "Status": { - "LastSeen": "2021-12-24T16:12:05.994305438+01:00", - "Connected": false - } - }, - "Ok+5QMdt/UjoktNOvicGYj+IX2g98p+0N2PJ3vJ45RI=": { - "Key": "Ok+5QMdt/UjoktNOvicGYj+IX2g98p+0N2PJ3vJ45RI=", - "SetupKey": "A72E4DC2-00DE-4542-8A24-62945438104E", - "IP": "100.64.0.1", - "Meta": { - "Hostname": "braginini", - "GoOS": "linux", - "Kernel": "Linux", - "Core": "21.04", - "Platform": "x86_64", - "OS": "Ubuntu", - "WtVersion": "" - }, - "Name": "braginini", - "Status": { - "LastSeen": "2021-12-24T16:11:27.015739803+01:00", - "Connected": false - } - } - } - } - } -} \ No newline at end of file diff --git a/management/server/testdata/storev1.sqlite b/management/server/testdata/storev1.sqlite new file mode 100644 index 0000000000000000000000000000000000000000..9a376698e4d226fc08fa68c12fb9bb4cf50375cd GIT binary patch literal 163840 zcmeI5U2GfKb;miPBucg=uCv)poJ29DU9U~Zv^o5y17WKV+N-Q3c_lkpV-XC8nW4`*J9@$Z(o!v!l_ zZtRO4rs=98wM402XSyyL>Mm1sv8yvplpBL{FD`FZ7VlRQtLrP3_Y&3PtEnb7))Upi zl~u1*oyMxkr(4r8zFg+Mzi3Fc78_h(aBAG?)BVMzyOp7BC9Y9#s|kbc8`r&Nol`|> zGdF8T(sx+HhT*2M{f?|jhO8=LLo!Izd^hAayI!qpj6OHS+JP(VrbtFmt|jS)iI%$H z6~4E*y0*CaF!4_1q20UW_2rHAt^1pctLyg@)$LtN7rNMyL_-&w)x_RxFITn_*3fh!OM^`&--(a&VtJT}48|!6gVSHBO~v>;&mE3=7G(B7H_b=w zDhAn%bw1s1BE539QCO+eQ@?yQ`E22QjOTgof|4 zJ;pe#CTwq{Go#xP0||1+HLNz7rMRQ&M$iDQ#HK2$dPr=SX$otFlIp7F8bQ@6)oDqF z8N@i zXg<}L7V3v1(9zUJw{C=HrZq$-zbz@C-rc2pZ6Vvg&C{)=^X7&+5Wx5y#dRXgK1 zW2bM{fn16l+JRVC8=j&1nX+*Z(yy1(WU7@Ba)+mIbJh&)kEkbia}!6oIlFcWqcn>q z_r_!VTeIBZCC>+W(>)kE{lr+mQ}Qzd!Pa(AepZj`d-mpNjGvw5KIScF<;(e>#=>sP z&p3te3Qm+j$QcspY%`~Eht@{jj=3ai^rUKRj~1jsvn5lfg&8fo$YXdhaL~f4Iv_-M zb*ah3mfV)9ZLjnI(>;Y5`NW>Xma=2x{GHihAv4f~;C~*kMPqz+hCAdut8yc;*ihS& zthlG6{nq4WIg+*=xrt;&b_LbsNB6iGKQqJqkX#)(5Ld!Ij)bkpO%B_RyBI!hdF-_5 z_+$G_BoD%xs^+b%X$|M(bTqjwk=t+@4Y)%$xX2Ci@A9_nJAXHaj;`N#lgPvOCfV^P zU@2x4AO#q*(xh(4y5&h@!nyqCYkalB%~PxgB!WAt|ZVQlAV%M9W@HqU|UJGB>~t z2wGl!wz#`6CYL|~Q6>+fYT*X@Cvx)U$T+_|Gir>MEljzqpVJw@*$)Bg~O zKcqLjKmY_l00ck)1V8`;KmY_l00cnb^FZJ*Ir7@p)Y{b4#Di~7O!+MS3kP%zgO&ch*1o@kS(a_y+gd>VUqQeN2C^Ak?-?nMQ+2wR9<$P8TFe zsMYI@LERKeLaFFk56<%@-UDkl|7sd90E5C8!X009sH0T2KI5CDNQPT*-|G(Nla;PtO=ZEa0VZe5(1$mY|zlKHQAg8Ph% z#Y`o)vYbh!(<_x!E}zS!N{jTobs09Z{jZC(bE7HT-<@vNgPi2z`RVt)1*`q{GP)krmxRoj1NDKL5PAFzec0=!f z=Wsu2A(&ZPTeA^la;1E^kShq;6ClVh-d-xCSC&!*;dUmK%jAVrsUireTscQ0s#Hp) zN_mBCOWl@XBG4H9YyF~3LW!Q9Kl<)*WhpOI%9W+mq9EK(*RNE|7^8ilo_q=D*9uFVpaS<1b}}Vx}l$jv2kM zlv&EBm(r=FY^9XSEv1)Ji@8i8mCxiWD@&zZNmyDr&g=u-&KL8UL0TUsGI;QzS^^?M z#-v;w=GH}PM`e;NO;Gw$dxJP-f@5C8!X009sH0T2KI5C8!X z_#-AT9i8R;D|VyiseknjFHd?`IPCrZuKE4{e~$kozWYZE0bK_H5C8!X z009sH0T2KI5C8!X0D;c~ffIgdfAoZM(YfdtPK})Z3;KW; z2!H?xfB*=900@A<871&+Wb`WAVh^rdroT&j>CzOvBJ}@aD+#)AQo~DYV=1LyC2Lgx72Qfezlu^zx$r^ zcSFsy553af2YNm|{FJ))@OZXt{dTzdlsdfx^C|VTkf0wHxBksr?^JfckB-yNdDC;g z=8ulg%{_}mhbZB4j}i*Ik8162Jr;J_-TJq_ySMb7n&0TwALch6?#(~i(wBrs5ANT6 zoGA+{PfntQ(^PkoXWM-xWd7hv=uv_H5WD%^0{cPc=TO1Z4@a)D)sxJF{nmc@W_hJF zFV~b!d3ANEq`ZA+ZU4^3clW;~6t&XU&b_yra!z`xFP&|r+~V)F0Gaxe_Kv3AJki0-6wZ8=jXS& z+Wt2W*0P;4X&j8SOOmf0=)3@*c zFGS)>d?Eg?@xM8v!4AU$0T2KI5C8!X009sH0T2KI5CDPSpFldwxYJzC95oA^=E_^t zEO45OMI$$(Z1OZ$H?aTz{nd|>AOHd&00JNY0w4eaAOHd&00Ms^1kTLc|3_YV@qHw{S00@8p2!H?xfB*=900@8p2!OzgCV>0@ zFPbpw2LTWO0T2KI5C8!X009sH0T2Lzmrnru|Cg@=^Z*1v00ck)1V8`;KmY_l00ck) z1YR@&Jpccq38Q`x009sH0T2KI5C8!X009sH0T6ik1aSZVxoLM`@bc@{}g)B8_WLt}_c+HKBCdOq1)^tBR>J)?!wJf%Krb7}UQx<(=8?YP8l; z>$|4ryjm@7w$!Gq1k~x-LRj;%u6NDCrsXA8Dz_IO+`XTelREO;^{T8Gtm#;m#0GZlby(P(tn>5z=F}_a`WPzR zV@`J`$N8%~Cwmra7u{tvVv;^LtZotrS866Ce-MjV7WbpJqcn0SwnR4+mh1Vmg+`V>nYEaJy}zgHdDx_+6u7BZObjcD?!{13cM|; z=A$+BE9!=1bj=KM#X4i5xf>LytBS(vjJ_asuX z=RHjKW=(a+mh_#VCFyj^6f>d*+Yhv92ciKY=D1Gilvrnnk%tU*i@cm&wKIM*-1%l5 z$fd}k9f)qq5a~gMOIn(W!tC&Vls>b$c(Hk^dGId%@(MpLth8Nfdt%a%s;%--$noMlT zZJFBkN)IsIQ<#xY>^W>HJ2uYWnH{#|2{a-2pT}#_7@wWt4mr=N-0&(k)V3rm?&)a1 zHMv=iq-{rTB3Y4LK{ff&Jub%2%y2&>S4R%Sm2i(EVe4^|!?xqDUyobSI&C`s*gg}< zgRrKmd5d9Mm^nEeO>Rr%Hk?KS?$8Y`a)bQ4ye<3A-_4<;>-XIx^6Mcobhn#gtN@}&#C&LiYT2_;2%S(aG4R8a3mRFxG z?yh#pB~U<=$%Cj`xPktOoV+!;h(7Ruk5|L z`+oSP%Q1dK;10{)#5Fyrwc()QJII~pDZ2G%&h<~|ej#^aSn<)^fcJ~^go~#i&ct{@ z;6A!yP4<5I{**-60rU%mU4cI+Z>|M)$_=L1HMwJMi-Xw?7;k#ofLsa34FsmY?H@ex zLAEPt^&Q8L=>2BaZ#2EzXxBFE1&F!z^A*>(tqu(O=-trr*T?y93&Sk4pRFkLr_FAE zEyfowafg$hanXe)!(2CdTXZ|&a=#JvN!oG2PVO<5XIH)w<1by}KDLd;ma*Sw!baf~ zPXw=*_?>a}==fpxv`2}%jCDk-qJW~QNMO}FzcUwh^ItvcDx_1rU?0E! z!KE0#c!fKh@rN(y4S!0t)C6;1(FXZ_n4?7&U6md7N z=9!meS1NZabS-yzacg;TrIIXud7OXe$}kt?)N31q5Ip;;@}(I64O)y|3|o%bMSrfl z?Oz&VGGXa}y--Gv`+0idi`K&Q*{*LaB;~fi4=@t5lH;7vX)VOoKiRl$UStsKJFLFz zD)x9SI%J%QW%Vz7Mb<}0+6rEjbxS&$Ox}F$s6~xSMYMHB4So6ipWBO??N@ulMc?|v zPWV#4&iW*6jX8cWd{OsQH!7aKanXt`-+$dwu3e%1{)Mn&owiP)*5K9C>vU7XykO|~ z2X{GXUZ?mBE}q(be-77v>U&0m@14A``#JB#$9b_l)R_?Z0k^m4lF6X-S0WLv_y@Z= zG@#+IYv{S67K|K!T=#PxU5W8GZ*qrGdr9AW+0|VFdDmgDh38jazKO|Nclnp6C0Z7l z7ao&`<8Q?HJ4x;^83+PF#~2C)!Hlc@&L!+p10~(C5G+q_Fz70uM(1LDGRZx2ZX|}h zbGYIw;qitu!!{8#f&t5iKnEOu^7}`F(AawJ(goT%IwpMM^>Kb9IduAkpqP;%*yx*w z=VvX1hwoa3WAC@!?RWU>jWEJ+rit*lUA^mfhi>DOha-s?fAuPNxNP-CR_Jn6i*93( zr`AI!uZQ~*%5$~pE%bxAw2?5cA#~jZVt?>X;`x8~3I>iK00JNY0w4eaAOHd&00JNY z0w8d93E=nt&aQ4TG7ta(5C8!X009sH0T2KI5C8!Xa0%f4ze@r~5C8!X009sH0T2KI z5C8!X009s{{QUi79#@z5C8!X009sH0T2KI5C8!X00EZ( zp8t1A;0OXB00JNY0w4eaAOHd&00JNY0%w;1?*E@%-C|@Q00JNY0w4eaAOHd&00JNY z0wCZL!2aJQfg=cj00@8p2!H?xfB*=900@8p2%KF4=JWqt{C5%hhZhKd00@8p2!H?x zfB*=900@8p2!O!ZBM{}Hktuxt|LiFiBLV>s009sH0T2KI5C8!X009sHfnEaG|MwEX zIS7CN2!H?xfB*=900@8p2!H?xoIL{g{{Pw2DMkbWAOHd&00JNY0w4eaAOHd&00O-P zu>bERf^!f60T2KI5C8!X009sH0T2KI5IB1T@cjST( Date: Fri, 4 Oct 2024 17:17:01 +0300 Subject: [PATCH 02/12] [management] Refactor User JWT group sync (#2690) * Refactor GetAccountIDByUserOrAccountID Signed-off-by: bcmmbaga * sync user jwt group changes Signed-off-by: bcmmbaga * propagate jwt group changes to peers Signed-off-by: bcmmbaga * fix no jwt groups synced Signed-off-by: bcmmbaga * fix tests and lint Signed-off-by: bcmmbaga * Move the account peer update outside the transaction Signed-off-by: bcmmbaga * move updateUserPeersInGroups to account manager Signed-off-by: bcmmbaga * move event store outside of transaction Signed-off-by: bcmmbaga * get user with update lock Signed-off-by: bcmmbaga * Run jwt sync in transaction Signed-off-by: bcmmbaga --------- Signed-off-by: bcmmbaga --- management/server/account.go | 284 ++++++++++++------ management/server/account_test.go | 169 +++++++---- management/server/mock_server/account_mock.go | 12 +- management/server/sql_store.go | 120 +++++++- management/server/sql_store_test.go | 30 ++ management/server/store.go | 5 +- management/server/user.go | 72 ++++- management/server/user_test.go | 5 +- 8 files changed, 520 insertions(+), 177 deletions(-) diff --git a/management/server/account.go b/management/server/account.go index d5e8c8cf8..da3203852 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -76,7 +76,7 @@ type AccountManager interface { SaveOrAddUsers(ctx context.Context, accountID, initiatorUserID string, updates []*User, addIfNotExists bool) ([]*UserInfo, error) GetSetupKey(ctx context.Context, accountID, userID, keyID string) (*SetupKey, error) GetAccountByID(ctx context.Context, accountID string, userID string) (*Account, error) - GetAccountIDByUserOrAccountID(ctx context.Context, userID, accountID, domain string) (string, error) + GetAccountIDByUserID(ctx context.Context, userID, domain string) (string, error) GetAccountIDFromToken(ctx context.Context, claims jwtclaims.AuthorizationClaims) (string, string, error) CheckUserAccessByJWTGroups(ctx context.Context, claims jwtclaims.AuthorizationClaims) error GetAccountFromPAT(ctx context.Context, pat string) (*Account, *User, *PersonalAccessToken, error) @@ -478,12 +478,12 @@ func (a *Account) GetPeerNetworkMap( } nm := &NetworkMap{ - Peers: peersToConnect, - Network: a.Network.Copy(), - Routes: routesUpdate, - DNSConfig: dnsUpdate, - OfflinePeers: expiredPeers, - FirewallRules: firewallRules, + Peers: peersToConnect, + Network: a.Network.Copy(), + Routes: routesUpdate, + DNSConfig: dnsUpdate, + OfflinePeers: expiredPeers, + FirewallRules: firewallRules, RoutesFirewallRules: routesFirewallRules, } @@ -843,55 +843,54 @@ func (a *Account) GetPeer(peerID string) *nbpeer.Peer { return a.Peers[peerID] } -// SetJWTGroups updates the user's auto groups by synchronizing JWT groups. -// Returns true if there are changes in the JWT group membership. -func (a *Account) SetJWTGroups(userID string, groupsNames []string) bool { - user, ok := a.Users[userID] - if !ok { - return false - } - +// getJWTGroupsChanges calculates the changes needed to sync a user's JWT groups. +// Returns a bool indicating if there are changes in the JWT group membership, the updated user AutoGroups, +// newly groups to create and an error if any occurred. +func (am *DefaultAccountManager) getJWTGroupsChanges(user *User, groups []*nbgroup.Group, groupNames []string) (bool, []string, []*nbgroup.Group, error) { existedGroupsByName := make(map[string]*nbgroup.Group) - for _, group := range a.Groups { + for _, group := range groups { existedGroupsByName[group.Name] = group } - newAutoGroups, jwtGroupsMap := separateGroups(user.AutoGroups, a.Groups) - groupsToAdd := difference(groupsNames, maps.Keys(jwtGroupsMap)) - groupsToRemove := difference(maps.Keys(jwtGroupsMap), groupsNames) + newUserAutoGroups, jwtGroupsMap := separateGroups(user.AutoGroups, groups) + + groupsToAdd := difference(groupNames, maps.Keys(jwtGroupsMap)) + groupsToRemove := difference(maps.Keys(jwtGroupsMap), groupNames) // If no groups are added or removed, we should not sync account if len(groupsToAdd) == 0 && len(groupsToRemove) == 0 { - return false + return false, nil, nil, nil } + newGroupsToCreate := make([]*nbgroup.Group, 0) + var modified bool for _, name := range groupsToAdd { group, exists := existedGroupsByName[name] if !exists { group = &nbgroup.Group{ - ID: xid.New().String(), - Name: name, - Issued: nbgroup.GroupIssuedJWT, + ID: xid.New().String(), + AccountID: user.AccountID, + Name: name, + Issued: nbgroup.GroupIssuedJWT, } - a.Groups[group.ID] = group + newGroupsToCreate = append(newGroupsToCreate, group) } if group.Issued == nbgroup.GroupIssuedJWT { - newAutoGroups = append(newAutoGroups, group.ID) + newUserAutoGroups = append(newUserAutoGroups, group.ID) modified = true } } for name, id := range jwtGroupsMap { if !slices.Contains(groupsToRemove, name) { - newAutoGroups = append(newAutoGroups, id) + newUserAutoGroups = append(newUserAutoGroups, id) continue } modified = true } - user.AutoGroups = newAutoGroups - return modified + return modified, newUserAutoGroups, newGroupsToCreate, nil } // UserGroupsAddToPeers adds groups to all peers of user @@ -1262,37 +1261,31 @@ func (am *DefaultAccountManager) DeleteAccount(ctx context.Context, accountID, u return nil } -// GetAccountIDByUserOrAccountID retrieves the account ID based on either the userID or accountID provided. -// If an accountID is provided, it checks if the account exists and returns it. -// If no accountID is provided, but a userID is given, it tries to retrieve the account by userID. +// GetAccountIDByUserID retrieves the account ID based on the userID provided. +// If user does have an account, it returns the user's account ID. // If the user doesn't have an account, it creates one using the provided domain. // Returns the account ID or an error if none is found or created. -func (am *DefaultAccountManager) GetAccountIDByUserOrAccountID(ctx context.Context, userID, accountID, domain string) (string, error) { - if accountID != "" { - exists, err := am.Store.AccountExists(ctx, LockingStrengthShare, accountID) - if err != nil { - return "", err - } - if !exists { - return "", status.Errorf(status.NotFound, "account %s does not exist", accountID) - } - return accountID, nil +func (am *DefaultAccountManager) GetAccountIDByUserID(ctx context.Context, userID, domain string) (string, error) { + if userID == "" { + return "", status.Errorf(status.NotFound, "no valid userID provided") } - if userID != "" { - account, err := am.GetOrCreateAccountByUser(ctx, userID, domain) - if err != nil { - return "", status.Errorf(status.NotFound, "account not found or created for user id: %s", userID) - } + accountID, err := am.Store.GetAccountIDByUserID(userID) + if err != nil { + if s, ok := status.FromError(err); ok && s.Type() == status.NotFound { + account, err := am.GetOrCreateAccountByUser(ctx, userID, domain) + if err != nil { + return "", status.Errorf(status.NotFound, "account not found or created for user id: %s", userID) + } - if err = am.addAccountIDToIDPAppMeta(ctx, userID, account); err != nil { - return "", err + if err = am.addAccountIDToIDPAppMeta(ctx, userID, account); err != nil { + return "", err + } + return account.Id, nil } - - return account.Id, nil + return "", err } - - return "", status.Errorf(status.NotFound, "no valid userID or accountID provided") + return accountID, nil } func isNil(i idp.Manager) bool { @@ -1796,6 +1789,10 @@ func (am *DefaultAccountManager) GetAccountIDFromToken(ctx context.Context, clai return "", "", status.Errorf(status.NotFound, "user %s not found", claims.UserId) } + if user.AccountID != accountID { + return "", "", status.Errorf(status.PermissionDenied, "user %s is not part of the account %s", claims.UserId, accountID) + } + if !user.IsServiceUser && claims.Invited { err = am.redeemInvite(ctx, accountID, user.Id) if err != nil { @@ -1803,7 +1800,7 @@ func (am *DefaultAccountManager) GetAccountIDFromToken(ctx context.Context, clai } } - if err = am.syncJWTGroups(ctx, accountID, user, claims); err != nil { + if err = am.syncJWTGroups(ctx, accountID, claims); err != nil { return "", "", err } @@ -1812,7 +1809,7 @@ func (am *DefaultAccountManager) GetAccountIDFromToken(ctx context.Context, clai // syncJWTGroups processes the JWT groups for a user, updates the account based on the groups, // and propagates changes to peers if group propagation is enabled. -func (am *DefaultAccountManager) syncJWTGroups(ctx context.Context, accountID string, user *User, claims jwtclaims.AuthorizationClaims) error { +func (am *DefaultAccountManager) syncJWTGroups(ctx context.Context, accountID string, claims jwtclaims.AuthorizationClaims) error { settings, err := am.Store.GetAccountSettings(ctx, LockingStrengthShare, accountID) if err != nil { return err @@ -1823,69 +1820,136 @@ func (am *DefaultAccountManager) syncJWTGroups(ctx context.Context, accountID st } if settings.JWTGroupsClaimName == "" { - log.WithContext(ctx).Errorf("JWT groups are enabled but no claim name is set") + log.WithContext(ctx).Debugf("JWT groups are enabled but no claim name is set") return nil } - // TODO: Remove GetAccount after refactoring account peer's update - unlock := am.Store.AcquireWriteLockByUID(ctx, accountID) - defer unlock() - - account, err := am.Store.GetAccount(ctx, accountID) - if err != nil { - return err - } - jwtGroupsNames := extractJWTGroups(ctx, settings.JWTGroupsClaimName, claims) - oldGroups := make([]string, len(user.AutoGroups)) - copy(oldGroups, user.AutoGroups) + unlockPeer := am.Store.AcquireWriteLockByUID(ctx, accountID) + defer func() { + if unlockPeer != nil { + unlockPeer() + } + }() - // Update the account if group membership changes - if account.SetJWTGroups(claims.UserId, jwtGroupsNames) { - addNewGroups := difference(user.AutoGroups, oldGroups) - removeOldGroups := difference(oldGroups, user.AutoGroups) - - if settings.GroupsPropagationEnabled { - account.UserGroupsAddToPeers(claims.UserId, addNewGroups...) - account.UserGroupsRemoveFromPeers(claims.UserId, removeOldGroups...) - account.Network.IncSerial() + var addNewGroups []string + var removeOldGroups []string + var hasChanges bool + var user *User + err = am.Store.ExecuteInTransaction(ctx, func(transaction Store) error { + user, err = am.Store.GetUserByUserID(ctx, LockingStrengthShare, claims.UserId) + if err != nil { + return fmt.Errorf("error getting user: %w", err) } - if err := am.Store.SaveAccount(ctx, account); err != nil { - log.WithContext(ctx).Errorf("failed to save account: %v", err) + groups, err := am.Store.GetAccountGroups(ctx, accountID) + if err != nil { + return fmt.Errorf("error getting account groups: %w", err) + } + + changed, updatedAutoGroups, newGroupsToCreate, err := am.getJWTGroupsChanges(user, groups, jwtGroupsNames) + if err != nil { + return fmt.Errorf("error getting JWT groups changes: %w", err) + } + + hasChanges = changed + // skip update if no changes + if !changed { return nil } + if err = transaction.SaveGroups(ctx, LockingStrengthUpdate, newGroupsToCreate); err != nil { + return fmt.Errorf("error saving groups: %w", err) + } + + addNewGroups = difference(updatedAutoGroups, user.AutoGroups) + removeOldGroups = difference(user.AutoGroups, updatedAutoGroups) + + user.AutoGroups = updatedAutoGroups + if err = transaction.SaveUser(ctx, LockingStrengthUpdate, user); err != nil { + return fmt.Errorf("error saving user: %w", err) + } + // Propagate changes to peers if group propagation is enabled if settings.GroupsPropagationEnabled { - log.WithContext(ctx).Tracef("user %s: JWT group membership changed, updating account peers", claims.UserId) - am.updateAccountPeers(ctx, account) - } + groups, err = transaction.GetAccountGroups(ctx, accountID) + if err != nil { + return fmt.Errorf("error getting account groups: %w", err) + } - for _, g := range addNewGroups { - if group := account.GetGroup(g); group != nil { - am.StoreEvent(ctx, user.Id, user.Id, account.Id, activity.GroupAddedToUser, - map[string]any{ - "group": group.Name, - "group_id": group.ID, - "is_service_user": user.IsServiceUser, - "user_name": user.ServiceUserName}) + groupsMap := make(map[string]*nbgroup.Group, len(groups)) + for _, group := range groups { + groupsMap[group.ID] = group + } + + peers, err := transaction.GetUserPeers(ctx, LockingStrengthShare, accountID, claims.UserId) + if err != nil { + return fmt.Errorf("error getting user peers: %w", err) + } + + updatedGroups, err := am.updateUserPeersInGroups(groupsMap, peers, addNewGroups, removeOldGroups) + if err != nil { + return fmt.Errorf("error modifying user peers in groups: %w", err) + } + + if err = transaction.SaveGroups(ctx, LockingStrengthUpdate, updatedGroups); err != nil { + return fmt.Errorf("error saving groups: %w", err) + } + + if err = transaction.IncrementNetworkSerial(ctx, accountID); err != nil { + return fmt.Errorf("error incrementing network serial: %w", err) } } + unlockPeer() + unlockPeer = nil - for _, g := range removeOldGroups { - if group := account.GetGroup(g); group != nil { - am.StoreEvent(ctx, user.Id, user.Id, account.Id, activity.GroupRemovedFromUser, - map[string]any{ - "group": group.Name, - "group_id": group.ID, - "is_service_user": user.IsServiceUser, - "user_name": user.ServiceUserName}) + return nil + }) + if err != nil { + return err + } + + if !hasChanges { + return nil + } + + for _, g := range addNewGroups { + group, err := am.Store.GetGroupByID(ctx, LockingStrengthShare, g, accountID) + if err != nil { + log.WithContext(ctx).Debugf("group %s not found while saving user activity event of account %s", g, accountID) + } else { + meta := map[string]any{ + "group": group.Name, "group_id": group.ID, + "is_service_user": user.IsServiceUser, "user_name": user.ServiceUserName, } + am.StoreEvent(ctx, user.Id, user.Id, accountID, activity.GroupAddedToUser, meta) } } + for _, g := range removeOldGroups { + group, err := am.Store.GetGroupByID(ctx, LockingStrengthShare, g, accountID) + if err != nil { + log.WithContext(ctx).Debugf("group %s not found while saving user activity event of account %s", g, accountID) + } else { + meta := map[string]any{ + "group": group.Name, "group_id": group.ID, + "is_service_user": user.IsServiceUser, "user_name": user.ServiceUserName, + } + am.StoreEvent(ctx, user.Id, user.Id, accountID, activity.GroupRemovedFromUser, meta) + } + } + + if settings.GroupsPropagationEnabled { + account, err := am.requestBuffer.GetAccountWithBackpressure(ctx, accountID) + if err != nil { + return fmt.Errorf("error getting account: %w", err) + } + + log.WithContext(ctx).Tracef("user %s: JWT group membership changed, updating account peers", claims.UserId) + am.updateAccountPeers(ctx, account) + } + return nil } @@ -1916,7 +1980,17 @@ func (am *DefaultAccountManager) getAccountIDWithAuthorizationClaims(ctx context // if Account ID is part of the claims // it means that we've already classified the domain and user has an account if claims.DomainCategory != PrivateCategory || !isDomainValid(claims.Domain) { - return am.GetAccountIDByUserOrAccountID(ctx, claims.UserId, claims.AccountId, claims.Domain) + if claims.AccountId != "" { + exists, err := am.Store.AccountExists(ctx, LockingStrengthShare, claims.AccountId) + if err != nil { + return "", err + } + if !exists { + return "", status.Errorf(status.NotFound, "account %s does not exist", claims.AccountId) + } + return claims.AccountId, nil + } + return am.GetAccountIDByUserID(ctx, claims.UserId, claims.Domain) } else if claims.AccountId != "" { userAccountID, err := am.Store.GetAccountIDByUserID(claims.UserId) if err != nil { @@ -2229,7 +2303,11 @@ func newAccountWithId(ctx context.Context, accountID, userID, domain string) *Ac routes := make(map[route.ID]*route.Route) setupKeys := map[string]*SetupKey{} nameServersGroups := make(map[string]*nbdns.NameServerGroup) - users[userID] = NewOwnerUser(userID) + + owner := NewOwnerUser(userID) + owner.AccountID = accountID + users[userID] = owner + dnsSettings := DNSSettings{ DisabledManagementGroups: make([]string, 0), } @@ -2297,12 +2375,17 @@ func userHasAllowedGroup(allowedGroups []string, userGroups []string) bool { // separateGroups separates user's auto groups into non-JWT and JWT groups. // Returns the list of standard auto groups and a map of JWT auto groups, // where the keys are the group names and the values are the group IDs. -func separateGroups(autoGroups []string, allGroups map[string]*nbgroup.Group) ([]string, map[string]string) { +func separateGroups(autoGroups []string, allGroups []*nbgroup.Group) ([]string, map[string]string) { newAutoGroups := make([]string, 0) jwtAutoGroups := make(map[string]string) // map of group name to group ID + allGroupsMap := make(map[string]*nbgroup.Group, len(allGroups)) + for _, group := range allGroups { + allGroupsMap[group.ID] = group + } + for _, id := range autoGroups { - if group, ok := allGroups[id]; ok { + if group, ok := allGroupsMap[id]; ok { if group.Issued == nbgroup.GroupIssuedJWT { jwtAutoGroups[group.Name] = id } else { @@ -2310,5 +2393,6 @@ func separateGroups(autoGroups []string, allGroups map[string]*nbgroup.Group) ([ } } } + return newAutoGroups, jwtAutoGroups } diff --git a/management/server/account_test.go b/management/server/account_test.go index 198775bc3..c417e4bc8 100644 --- a/management/server/account_test.go +++ b/management/server/account_test.go @@ -633,7 +633,7 @@ func TestDefaultAccountManager_GetAccountIDFromToken(t *testing.T) { manager, err := createManager(t) require.NoError(t, err, "unable to create account manager") - accountID, err := manager.GetAccountIDByUserOrAccountID(context.Background(), testCase.inputInitUserParams.UserId, testCase.inputInitUserParams.AccountId, testCase.inputInitUserParams.Domain) + accountID, err := manager.GetAccountIDByUserID(context.Background(), testCase.inputInitUserParams.UserId, testCase.inputInitUserParams.Domain) require.NoError(t, err, "create init user failed") initAccount, err := manager.Store.GetAccount(context.Background(), accountID) @@ -671,17 +671,16 @@ func TestDefaultAccountManager_GetGroupsFromTheToken(t *testing.T) { userId := "user-id" domain := "test.domain" - initAccount := newAccountWithId(context.Background(), "", userId, domain) + _ = newAccountWithId(context.Background(), "", userId, domain) manager, err := createManager(t) require.NoError(t, err, "unable to create account manager") - accountID := initAccount.Id - accountID, err = manager.GetAccountIDByUserOrAccountID(context.Background(), userId, accountID, domain) + accountID, err := manager.GetAccountIDByUserID(context.Background(), userId, domain) require.NoError(t, err, "create init user failed") // as initAccount was created without account id we have to take the id after account initialization - // that happens inside the GetAccountIDByUserOrAccountID where the id is getting generated + // that happens inside the GetAccountIDByUserID where the id is getting generated // it is important to set the id as it help to avoid creating additional account with empty Id and re-pointing indices to it - initAccount, err = manager.Store.GetAccount(context.Background(), accountID) + initAccount, err := manager.Store.GetAccount(context.Background(), accountID) require.NoError(t, err, "get init account failed") claims := jwtclaims.AuthorizationClaims{ @@ -885,7 +884,7 @@ func TestAccountManager_SetOrUpdateDomain(t *testing.T) { } } -func TestAccountManager_GetAccountByUserOrAccountId(t *testing.T) { +func TestAccountManager_GetAccountByUserID(t *testing.T) { manager, err := createManager(t) if err != nil { t.Fatal(err) @@ -894,7 +893,7 @@ func TestAccountManager_GetAccountByUserOrAccountId(t *testing.T) { userId := "test_user" - accountID, err := manager.GetAccountIDByUserOrAccountID(context.Background(), userId, "", "") + accountID, err := manager.GetAccountIDByUserID(context.Background(), userId, "") if err != nil { t.Fatal(err) } @@ -903,14 +902,13 @@ func TestAccountManager_GetAccountByUserOrAccountId(t *testing.T) { return } - _, err = manager.GetAccountIDByUserOrAccountID(context.Background(), "", accountID, "") - if err != nil { - t.Errorf("expected to get existing account after creation using userid, no account was found for a account %s", accountID) - } + exists, err := manager.Store.AccountExists(context.Background(), LockingStrengthShare, accountID) + assert.NoError(t, err) + assert.True(t, exists, "expected to get existing account after creation using userid") - _, err = manager.GetAccountIDByUserOrAccountID(context.Background(), "", "", "") + _, err = manager.GetAccountIDByUserID(context.Background(), "", "") if err == nil { - t.Errorf("expected an error when user and account IDs are empty") + t.Errorf("expected an error when user ID is empty") } } @@ -1669,7 +1667,7 @@ func TestDefaultAccountManager_DefaultAccountSettings(t *testing.T) { manager, err := createManager(t) require.NoError(t, err, "unable to create account manager") - accountID, err := manager.GetAccountIDByUserOrAccountID(context.Background(), userID, "", "") + accountID, err := manager.GetAccountIDByUserID(context.Background(), userID, "") require.NoError(t, err, "unable to create an account") settings, err := manager.Store.GetAccountSettings(context.Background(), LockingStrengthShare, accountID) @@ -1684,7 +1682,7 @@ func TestDefaultAccountManager_UpdatePeer_PeerLoginExpiration(t *testing.T) { manager, err := createManager(t) require.NoError(t, err, "unable to create account manager") - _, err = manager.GetAccountIDByUserOrAccountID(context.Background(), userID, "", "") + _, err = manager.GetAccountIDByUserID(context.Background(), userID, "") require.NoError(t, err, "unable to create an account") key, err := wgtypes.GenerateKey() @@ -1696,7 +1694,7 @@ func TestDefaultAccountManager_UpdatePeer_PeerLoginExpiration(t *testing.T) { }) require.NoError(t, err, "unable to add peer") - accountID, err := manager.GetAccountIDByUserOrAccountID(context.Background(), userID, "", "") + accountID, err := manager.GetAccountIDByUserID(context.Background(), userID, "") require.NoError(t, err, "unable to get the account") account, err := manager.Store.GetAccount(context.Background(), accountID) @@ -1742,7 +1740,7 @@ func TestDefaultAccountManager_MarkPeerConnected_PeerLoginExpiration(t *testing. manager, err := createManager(t) require.NoError(t, err, "unable to create account manager") - accountID, err := manager.GetAccountIDByUserOrAccountID(context.Background(), userID, "", "") + accountID, err := manager.GetAccountIDByUserID(context.Background(), userID, "") require.NoError(t, err, "unable to create an account") key, err := wgtypes.GenerateKey() @@ -1770,7 +1768,7 @@ func TestDefaultAccountManager_MarkPeerConnected_PeerLoginExpiration(t *testing. }, } - accountID, err = manager.GetAccountIDByUserOrAccountID(context.Background(), userID, "", "") + accountID, err = manager.GetAccountIDByUserID(context.Background(), userID, "") require.NoError(t, err, "unable to get the account") account, err := manager.Store.GetAccount(context.Background(), accountID) @@ -1790,7 +1788,7 @@ func TestDefaultAccountManager_UpdateAccountSettings_PeerLoginExpiration(t *test manager, err := createManager(t) require.NoError(t, err, "unable to create account manager") - _, err = manager.GetAccountIDByUserOrAccountID(context.Background(), userID, "", "") + _, err = manager.GetAccountIDByUserID(context.Background(), userID, "") require.NoError(t, err, "unable to create an account") key, err := wgtypes.GenerateKey() @@ -1802,7 +1800,7 @@ func TestDefaultAccountManager_UpdateAccountSettings_PeerLoginExpiration(t *test }) require.NoError(t, err, "unable to add peer") - accountID, err := manager.GetAccountIDByUserOrAccountID(context.Background(), userID, "", "") + accountID, err := manager.GetAccountIDByUserID(context.Background(), userID, "") require.NoError(t, err, "unable to get the account") account, err := manager.Store.GetAccount(context.Background(), accountID) @@ -1850,7 +1848,7 @@ func TestDefaultAccountManager_UpdateAccountSettings(t *testing.T) { manager, err := createManager(t) require.NoError(t, err, "unable to create account manager") - accountID, err := manager.GetAccountIDByUserOrAccountID(context.Background(), userID, "", "") + accountID, err := manager.GetAccountIDByUserID(context.Background(), userID, "") require.NoError(t, err, "unable to create an account") updated, err := manager.UpdateAccountSettings(context.Background(), accountID, userID, &Settings{ @@ -1861,9 +1859,6 @@ func TestDefaultAccountManager_UpdateAccountSettings(t *testing.T) { assert.False(t, updated.Settings.PeerLoginExpirationEnabled) assert.Equal(t, updated.Settings.PeerLoginExpiration, time.Hour) - accountID, err = manager.GetAccountIDByUserOrAccountID(context.Background(), "", accountID, "") - require.NoError(t, err, "unable to get account by ID") - settings, err := manager.Store.GetAccountSettings(context.Background(), LockingStrengthShare, accountID) require.NoError(t, err, "unable to get account settings") @@ -2199,8 +2194,12 @@ func TestAccount_GetNextPeerExpiration(t *testing.T) { } func TestAccount_SetJWTGroups(t *testing.T) { + manager, err := createManager(t) + require.NoError(t, err, "unable to create account manager") + // create a new account account := &Account{ + Id: "accountID", Peers: map[string]*nbpeer.Peer{ "peer1": {ID: "peer1", Key: "key1", UserID: "user1"}, "peer2": {ID: "peer2", Key: "key2", UserID: "user1"}, @@ -2211,62 +2210,120 @@ func TestAccount_SetJWTGroups(t *testing.T) { Groups: map[string]*group.Group{ "group1": {ID: "group1", Name: "group1", Issued: group.GroupIssuedAPI, Peers: []string{}}, }, - Settings: &Settings{GroupsPropagationEnabled: true}, + Settings: &Settings{GroupsPropagationEnabled: true, JWTGroupsEnabled: true, JWTGroupsClaimName: "groups"}, Users: map[string]*User{ - "user1": {Id: "user1"}, - "user2": {Id: "user2"}, + "user1": {Id: "user1", AccountID: "accountID"}, + "user2": {Id: "user2", AccountID: "accountID"}, }, } + assert.NoError(t, manager.Store.SaveAccount(context.Background(), account), "unable to save account") + t.Run("empty jwt groups", func(t *testing.T) { - updated := account.SetJWTGroups("user1", []string{}) - assert.False(t, updated, "account should not be updated") - assert.Empty(t, account.Users["user1"].AutoGroups, "auto groups must be empty") + claims := jwtclaims.AuthorizationClaims{ + UserId: "user1", + Raw: jwt.MapClaims{"groups": []interface{}{}}, + } + err := manager.syncJWTGroups(context.Background(), "accountID", claims) + assert.NoError(t, err, "unable to sync jwt groups") + + user, err := manager.Store.GetUserByUserID(context.Background(), LockingStrengthShare, "user1") + assert.NoError(t, err, "unable to get user") + assert.Empty(t, user.AutoGroups, "auto groups must be empty") }) t.Run("jwt match existing api group", func(t *testing.T) { - updated := account.SetJWTGroups("user1", []string{"group1"}) - assert.False(t, updated, "account should not be updated") - assert.Equal(t, 0, len(account.Users["user1"].AutoGroups)) - assert.Equal(t, account.Groups["group1"].Issued, group.GroupIssuedAPI, "group should be api issued") + claims := jwtclaims.AuthorizationClaims{ + UserId: "user1", + Raw: jwt.MapClaims{"groups": []interface{}{"group1"}}, + } + err := manager.syncJWTGroups(context.Background(), "accountID", claims) + assert.NoError(t, err, "unable to sync jwt groups") + + user, err := manager.Store.GetUserByUserID(context.Background(), LockingStrengthShare, "user1") + assert.NoError(t, err, "unable to get user") + assert.Len(t, user.AutoGroups, 0) + + group1, err := manager.Store.GetGroupByID(context.Background(), LockingStrengthShare, "group1", "accountID") + assert.NoError(t, err, "unable to get group") + assert.Equal(t, group1.Issued, group.GroupIssuedAPI, "group should be api issued") }) t.Run("jwt match existing api group in user auto groups", func(t *testing.T) { account.Users["user1"].AutoGroups = []string{"group1"} + assert.NoError(t, manager.Store.SaveUser(context.Background(), LockingStrengthUpdate, account.Users["user1"])) - updated := account.SetJWTGroups("user1", []string{"group1"}) - assert.False(t, updated, "account should not be updated") - assert.Equal(t, 1, len(account.Users["user1"].AutoGroups)) - assert.Equal(t, account.Groups["group1"].Issued, group.GroupIssuedAPI, "group should be api issued") + claims := jwtclaims.AuthorizationClaims{ + UserId: "user1", + Raw: jwt.MapClaims{"groups": []interface{}{"group1"}}, + } + err = manager.syncJWTGroups(context.Background(), "accountID", claims) + assert.NoError(t, err, "unable to sync jwt groups") + + user, err := manager.Store.GetUserByUserID(context.Background(), LockingStrengthShare, "user1") + assert.NoError(t, err, "unable to get user") + assert.Len(t, user.AutoGroups, 1) + + group1, err := manager.Store.GetGroupByID(context.Background(), LockingStrengthShare, "group1", "accountID") + assert.NoError(t, err, "unable to get group") + assert.Equal(t, group1.Issued, group.GroupIssuedAPI, "group should be api issued") }) t.Run("add jwt group", func(t *testing.T) { - updated := account.SetJWTGroups("user1", []string{"group1", "group2"}) - assert.True(t, updated, "account should be updated") - assert.Len(t, account.Groups, 2, "new group should be added") - assert.Len(t, account.Users["user1"].AutoGroups, 2, "new group should be added") - assert.Contains(t, account.Groups, account.Users["user1"].AutoGroups[0], "groups must contain group2 from user groups") + claims := jwtclaims.AuthorizationClaims{ + UserId: "user1", + Raw: jwt.MapClaims{"groups": []interface{}{"group1", "group2"}}, + } + err = manager.syncJWTGroups(context.Background(), "accountID", claims) + assert.NoError(t, err, "unable to sync jwt groups") + + user, err := manager.Store.GetUserByUserID(context.Background(), LockingStrengthShare, "user1") + assert.NoError(t, err, "unable to get user") + assert.Len(t, user.AutoGroups, 2, "groups count should not be change") }) t.Run("existed group not update", func(t *testing.T) { - updated := account.SetJWTGroups("user1", []string{"group2"}) - assert.False(t, updated, "account should not be updated") - assert.Len(t, account.Groups, 2, "groups count should not be changed") + claims := jwtclaims.AuthorizationClaims{ + UserId: "user1", + Raw: jwt.MapClaims{"groups": []interface{}{"group2"}}, + } + err = manager.syncJWTGroups(context.Background(), "accountID", claims) + assert.NoError(t, err, "unable to sync jwt groups") + + user, err := manager.Store.GetUserByUserID(context.Background(), LockingStrengthShare, "user1") + assert.NoError(t, err, "unable to get user") + assert.Len(t, user.AutoGroups, 2, "groups count should not be change") }) t.Run("add new group", func(t *testing.T) { - updated := account.SetJWTGroups("user2", []string{"group1", "group3"}) - assert.True(t, updated, "account should be updated") - assert.Len(t, account.Groups, 3, "new group should be added") - assert.Len(t, account.Users["user2"].AutoGroups, 1, "new group should be added") - assert.Contains(t, account.Groups, account.Users["user2"].AutoGroups[0], "groups must contain group3 from user groups") + claims := jwtclaims.AuthorizationClaims{ + UserId: "user2", + Raw: jwt.MapClaims{"groups": []interface{}{"group1", "group3"}}, + } + err = manager.syncJWTGroups(context.Background(), "accountID", claims) + assert.NoError(t, err, "unable to sync jwt groups") + + groups, err := manager.Store.GetAccountGroups(context.Background(), "accountID") + assert.NoError(t, err) + assert.Len(t, groups, 3, "new group3 should be added") + + user, err := manager.Store.GetUserByUserID(context.Background(), LockingStrengthShare, "user2") + assert.NoError(t, err, "unable to get user") + assert.Len(t, user.AutoGroups, 1, "new group should be added") }) t.Run("remove all JWT groups", func(t *testing.T) { - updated := account.SetJWTGroups("user1", []string{}) - assert.True(t, updated, "account should be updated") - assert.Len(t, account.Users["user1"].AutoGroups, 1, "only non-JWT groups should remain") - assert.Contains(t, account.Users["user1"].AutoGroups, "group1", " group1 should still be present") + claims := jwtclaims.AuthorizationClaims{ + UserId: "user1", + Raw: jwt.MapClaims{"groups": []interface{}{}}, + } + err = manager.syncJWTGroups(context.Background(), "accountID", claims) + assert.NoError(t, err, "unable to sync jwt groups") + + user, err := manager.Store.GetUserByUserID(context.Background(), LockingStrengthShare, "user1") + assert.NoError(t, err, "unable to get user") + assert.Len(t, user.AutoGroups, 1, "only non-JWT groups should remain") + assert.Contains(t, user.AutoGroups, "group1", " group1 should still be present") }) } diff --git a/management/server/mock_server/account_mock.go b/management/server/mock_server/account_mock.go index b399be822..b6283a7e6 100644 --- a/management/server/mock_server/account_mock.go +++ b/management/server/mock_server/account_mock.go @@ -27,7 +27,7 @@ type MockAccountManager struct { CreateSetupKeyFunc func(ctx context.Context, accountId string, keyName string, keyType server.SetupKeyType, expiresIn time.Duration, autoGroups []string, usageLimit int, userID string, ephemeral bool) (*server.SetupKey, error) GetSetupKeyFunc func(ctx context.Context, accountID, userID, keyID string) (*server.SetupKey, error) - GetAccountIDByUserOrAccountIdFunc func(ctx context.Context, userId, accountId, domain string) (string, error) + GetAccountIDByUserIdFunc func(ctx context.Context, userId, domain string) (string, error) GetUserFunc func(ctx context.Context, claims jwtclaims.AuthorizationClaims) (*server.User, error) ListUsersFunc func(ctx context.Context, accountID string) ([]*server.User, error) GetPeersFunc func(ctx context.Context, accountID, userID string) ([]*nbpeer.Peer, error) @@ -194,14 +194,14 @@ func (am *MockAccountManager) CreateSetupKey( return nil, status.Errorf(codes.Unimplemented, "method CreateSetupKey is not implemented") } -// GetAccountIDByUserOrAccountID mock implementation of GetAccountIDByUserOrAccountID from server.AccountManager interface -func (am *MockAccountManager) GetAccountIDByUserOrAccountID(ctx context.Context, userId, accountId, domain string) (string, error) { - if am.GetAccountIDByUserOrAccountIdFunc != nil { - return am.GetAccountIDByUserOrAccountIdFunc(ctx, userId, accountId, domain) +// GetAccountIDByUserID mock implementation of GetAccountIDByUserID from server.AccountManager interface +func (am *MockAccountManager) GetAccountIDByUserID(ctx context.Context, userId, domain string) (string, error) { + if am.GetAccountIDByUserIdFunc != nil { + return am.GetAccountIDByUserIdFunc(ctx, userId, domain) } return "", status.Errorf( codes.Unimplemented, - "method GetAccountIDByUserOrAccountID is not implemented", + "method GetAccountIDByUserID is not implemented", ) } diff --git a/management/server/sql_store.go b/management/server/sql_store.go index cce748a0f..9e1ab27dc 100644 --- a/management/server/sql_store.go +++ b/management/server/sql_store.go @@ -10,6 +10,7 @@ import ( "path/filepath" "runtime" "runtime/debug" + "slices" "strings" "sync" "time" @@ -378,15 +379,26 @@ func (s *SqlStore) SaveUsers(accountID string, users map[string]*User) error { Create(&usersToSave).Error } -// SaveGroups saves the given list of groups to the database. -// It updates existing groups if a conflict occurs. -func (s *SqlStore) SaveGroups(accountID string, groups map[string]*nbgroup.Group) error { - groupsToSave := make([]nbgroup.Group, 0, len(groups)) - for _, group := range groups { - group.AccountID = accountID - groupsToSave = append(groupsToSave, *group) +// SaveUser saves the given user to the database. +func (s *SqlStore) SaveUser(ctx context.Context, lockStrength LockingStrength, user *User) error { + result := s.db.WithContext(ctx).Clauses(clause.Locking{Strength: string(lockStrength)}).Save(user) + if result.Error != nil { + return status.Errorf(status.Internal, "failed to save user to store: %v", result.Error) } - return s.db.Clauses(clause.OnConflict{UpdateAll: true}).Create(&groupsToSave).Error + return nil +} + +// SaveGroups saves the given list of groups to the database. +func (s *SqlStore) SaveGroups(ctx context.Context, lockStrength LockingStrength, groups []*nbgroup.Group) error { + if len(groups) == 0 { + return nil + } + + result := s.db.WithContext(ctx).Clauses(clause.Locking{Strength: string(lockStrength)}).Save(&groups) + if result.Error != nil { + return status.Errorf(status.Internal, "failed to save groups to store: %v", result.Error) + } + return nil } // DeleteHashedPAT2TokenIDIndex is noop in SqlStore @@ -1021,6 +1033,89 @@ func (s *SqlStore) AddPeerToGroup(ctx context.Context, accountId string, peerId return nil } +// AddUserPeersToGroups adds the user's peers to specified groups in database. +func (s *SqlStore) AddUserPeersToGroups(ctx context.Context, accountID string, userID string, groupIDs []string) error { + if len(groupIDs) == 0 { + return nil + } + + var userPeerIDs []string + result := s.db.WithContext(ctx).Clauses(clause.Locking{Strength: string(LockingStrengthShare)}).Select("id"). + Where("account_id = ? AND user_id = ?", accountID, userID).Model(&nbpeer.Peer{}).Find(&userPeerIDs) + if result.Error != nil { + return status.Errorf(status.Internal, "issue finding user peers") + } + + groupsToUpdate := make([]*nbgroup.Group, 0, len(groupIDs)) + for _, gid := range groupIDs { + group, err := s.GetGroupByID(ctx, LockingStrengthShare, gid, accountID) + if err != nil { + return err + } + + groupPeers := make(map[string]struct{}) + for _, pid := range group.Peers { + groupPeers[pid] = struct{}{} + } + + for _, pid := range userPeerIDs { + groupPeers[pid] = struct{}{} + } + + group.Peers = group.Peers[:0] + for pid := range groupPeers { + group.Peers = append(group.Peers, pid) + } + + groupsToUpdate = append(groupsToUpdate, group) + } + + return s.SaveGroups(ctx, LockingStrengthUpdate, groupsToUpdate) +} + +// RemoveUserPeersFromGroups removes the user's peers from specified groups in database. +func (s *SqlStore) RemoveUserPeersFromGroups(ctx context.Context, accountID string, userID string, groupIDs []string) error { + if len(groupIDs) == 0 { + return nil + } + + var userPeerIDs []string + result := s.db.WithContext(ctx).Clauses(clause.Locking{Strength: string(LockingStrengthShare)}).Select("id"). + Where("account_id = ? AND user_id = ?", accountID, userID).Model(&nbpeer.Peer{}).Find(&userPeerIDs) + if result.Error != nil { + return status.Errorf(status.Internal, "issue finding user peers") + } + + groupsToUpdate := make([]*nbgroup.Group, 0, len(groupIDs)) + for _, gid := range groupIDs { + group, err := s.GetGroupByID(ctx, LockingStrengthShare, gid, accountID) + if err != nil { + return err + } + + if group.Name == "All" { + continue + } + + update := make([]string, 0, len(group.Peers)) + for _, pid := range group.Peers { + if !slices.Contains(userPeerIDs, pid) { + update = append(update, pid) + } + } + + group.Peers = update + groupsToUpdate = append(groupsToUpdate, group) + } + + return s.SaveGroups(ctx, LockingStrengthUpdate, groupsToUpdate) +} + +// GetUserPeers retrieves peers for a user. +func (s *SqlStore) GetUserPeers(ctx context.Context, lockStrength LockingStrength, accountID, userID string) ([]*nbpeer.Peer, error) { + return getRecords[*nbpeer.Peer](s.db.WithContext(ctx).Where("user_id = ?", userID), lockStrength, accountID) +} + func (s *SqlStore) AddPeerToAccount(ctx context.Context, peer *nbpeer.Peer) error { if err := s.db.WithContext(ctx).Create(peer).Error; err != nil { return status.Errorf(status.Internal, "issue adding peer to account") @@ -1127,6 +1222,15 @@ func (s *SqlStore) GetGroupByName(ctx context.Context, lockStrength LockingStren return &group, nil } +// SaveGroup saves a group to the store. +func (s *SqlStore) SaveGroup(ctx context.Context, lockStrength LockingStrength, group *nbgroup.Group) error { + result := s.db.WithContext(ctx).Clauses(clause.Locking{Strength: string(lockStrength)}).Save(group) + if result.Error != nil { + return status.Errorf(status.Internal, "failed to save group to store: %v", result.Error) + } + return nil +} + // GetAccountPolicies retrieves policies for an account. func (s *SqlStore) GetAccountPolicies(ctx context.Context, lockStrength LockingStrength, accountID string) ([]*Policy, error) { return getRecords[*Policy](s.db.WithContext(ctx).Preload(clause.Associations), lockStrength, accountID) diff --git a/management/server/sql_store_test.go b/management/server/sql_store_test.go index dc07849d9..4eed09c69 100644 --- a/management/server/sql_store_test.go +++ b/management/server/sql_store_test.go @@ -1185,3 +1185,33 @@ func TestSqlite_incrementSetupKeyUsage(t *testing.T) { require.NoError(t, err) assert.Equal(t, 2, setupKey.UsedTimes) } + +func TestSqlite_CreateAndGetObjectInTransaction(t *testing.T) { + store, cleanup, err := NewSqliteTestStore(context.Background(), t.TempDir(), "testdata/extended-store.sqlite") + t.Cleanup(cleanup) + if err != nil { + t.Fatal(err) + } + group := &nbgroup.Group{ + ID: "group-id", + AccountID: "account-id", + Name: "group-name", + Issued: "api", + Peers: nil, + } + err = store.ExecuteInTransaction(context.Background(), func(transaction Store) error { + err := transaction.SaveGroup(context.Background(), LockingStrengthUpdate, group) + if err != nil { + t.Fatal("failed to save group") + return err + } + group, err = transaction.GetGroupByID(context.Background(), LockingStrengthUpdate, group.ID, group.AccountID) + if err != nil { + t.Fatal("failed to get group") + return err + } + t.Logf("group: %v", group) + return nil + }) + assert.NoError(t, err) +} diff --git a/management/server/store.go b/management/server/store.go index 041c936ae..50bc6afdf 100644 --- a/management/server/store.go +++ b/management/server/store.go @@ -60,6 +60,7 @@ type Store interface { GetUserByTokenID(ctx context.Context, tokenID string) (*User, error) GetUserByUserID(ctx context.Context, lockStrength LockingStrength, userID string) (*User, error) SaveUsers(accountID string, users map[string]*User) error + SaveUser(ctx context.Context, lockStrength LockingStrength, user *User) error SaveUserLastLogin(ctx context.Context, accountID, userID string, lastLogin time.Time) error GetTokenIDByHashedToken(ctx context.Context, secret string) (string, error) DeleteHashedPAT2TokenIDIndex(hashedToken string) error @@ -68,7 +69,8 @@ type Store interface { GetAccountGroups(ctx context.Context, accountID string) ([]*nbgroup.Group, error) GetGroupByID(ctx context.Context, lockStrength LockingStrength, groupID, accountID string) (*nbgroup.Group, error) GetGroupByName(ctx context.Context, lockStrength LockingStrength, groupName, accountID string) (*nbgroup.Group, error) - SaveGroups(accountID string, groups map[string]*nbgroup.Group) error + SaveGroups(ctx context.Context, lockStrength LockingStrength, groups []*nbgroup.Group) error + SaveGroup(ctx context.Context, lockStrength LockingStrength, group *nbgroup.Group) error GetAccountPolicies(ctx context.Context, lockStrength LockingStrength, accountID string) ([]*Policy, error) GetPolicyByID(ctx context.Context, lockStrength LockingStrength, policyID string, accountID string) (*Policy, error) @@ -82,6 +84,7 @@ type Store interface { AddPeerToGroup(ctx context.Context, accountId string, peerId string, groupID string) error AddPeerToAccount(ctx context.Context, peer *nbpeer.Peer) error GetPeerByPeerPubKey(ctx context.Context, lockStrength LockingStrength, peerKey string) (*nbpeer.Peer, error) + GetUserPeers(ctx context.Context, lockStrength LockingStrength, accountID, userID string) ([]*nbpeer.Peer, error) SavePeer(ctx context.Context, accountID string, peer *nbpeer.Peer) error SavePeerStatus(accountID, peerID string, status nbpeer.PeerStatus) error SavePeerLocation(accountID string, peer *nbpeer.Peer) error diff --git a/management/server/user.go b/management/server/user.go index 6d01561c6..38a8ac0c4 100644 --- a/management/server/user.go +++ b/management/server/user.go @@ -8,14 +8,14 @@ import ( "time" "github.com/google/uuid" - log "github.com/sirupsen/logrus" - "github.com/netbirdio/netbird/management/server/activity" + nbgroup "github.com/netbirdio/netbird/management/server/group" "github.com/netbirdio/netbird/management/server/idp" "github.com/netbirdio/netbird/management/server/integration_reference" "github.com/netbirdio/netbird/management/server/jwtclaims" nbpeer "github.com/netbirdio/netbird/management/server/peer" "github.com/netbirdio/netbird/management/server/status" + log "github.com/sirupsen/logrus" ) const ( @@ -1254,6 +1254,74 @@ func (am *DefaultAccountManager) prepareUserDeletion(ctx context.Context, accoun return map[string]any{"name": tuName, "email": tuEmail, "created_at": tuCreatedAt}, nil } +// updateUserPeersInGroups updates the user's peers in the specified groups by adding or removing them. +func (am *DefaultAccountManager) updateUserPeersInGroups(accountGroups map[string]*nbgroup.Group, peers []*nbpeer.Peer, groupsToAdd, + groupsToRemove []string) (groupsToUpdate []*nbgroup.Group, err error) { + + if len(groupsToAdd) == 0 && len(groupsToRemove) == 0 { + return + } + + userPeerIDMap := make(map[string]struct{}, len(peers)) + for _, peer := range peers { + userPeerIDMap[peer.ID] = struct{}{} + } + + for _, gid := range groupsToAdd { + group, ok := accountGroups[gid] + if !ok { + return nil, errors.New("group not found") + } + addUserPeersToGroup(userPeerIDMap, group) + groupsToUpdate = append(groupsToUpdate, group) + } + + for _, gid := range groupsToRemove { + group, ok := accountGroups[gid] + if !ok { + return nil, errors.New("group not found") + } + removeUserPeersFromGroup(userPeerIDMap, group) + groupsToUpdate = append(groupsToUpdate, group) + } + + return groupsToUpdate, nil +} + +// addUserPeersToGroup adds the user's peers to the group. +func addUserPeersToGroup(userPeerIDs map[string]struct{}, group *nbgroup.Group) { + groupPeers := make(map[string]struct{}, len(group.Peers)) + for _, pid := range group.Peers { + groupPeers[pid] = struct{}{} + } + + for pid := range userPeerIDs { + groupPeers[pid] = struct{}{} + } + + group.Peers = make([]string, 0, len(groupPeers)) + for pid := range groupPeers { + group.Peers = append(group.Peers, pid) + } +} + +// removeUserPeersFromGroup removes user's peers from the group. +func removeUserPeersFromGroup(userPeerIDs map[string]struct{}, group *nbgroup.Group) { + // skip removing peers from group All + if group.Name == "All" { + return + } + + updatedPeers := make([]string, 0, len(group.Peers)) + for _, pid := range group.Peers { + if _, found := userPeerIDs[pid]; !found { + updatedPeers = append(updatedPeers, pid) + } + } + + group.Peers = updatedPeers +} + func findUserInIDPUserdata(userID string, userData []*idp.UserData) (*idp.UserData, bool) { for _, user := range userData { if user.ID == userID { diff --git a/management/server/user_test.go b/management/server/user_test.go index ec0a10695..1a5704551 100644 --- a/management/server/user_test.go +++ b/management/server/user_test.go @@ -813,10 +813,7 @@ func TestUser_DeleteUser_RegularUsers(t *testing.T) { assert.NoError(t, err) } - accID, err := am.GetAccountIDByUserOrAccountID(context.Background(), "", account.Id, "") - assert.NoError(t, err) - - acc, err := am.Store.GetAccount(context.Background(), accID) + acc, err := am.Store.GetAccount(context.Background(), account.Id) assert.NoError(t, err) for _, id := range tc.expectedDeleted { From 8bf729c7b4e8d3f7a7a88c9dc2ac60b63c803a8c Mon Sep 17 00:00:00 2001 From: Bethuel Mmbaga Date: Fri, 4 Oct 2024 18:09:40 +0300 Subject: [PATCH 03/12] [management] Add AccountExists to AccountManager (#2694) * Add AccountExists method to account manager interface Signed-off-by: bcmmbaga * remove unused code Signed-off-by: bcmmbaga --------- Signed-off-by: bcmmbaga --- management/server/account.go | 6 ++ management/server/mock_server/account_mock.go | 13 ++- management/server/sql_store.go | 79 ------------------- 3 files changed, 17 insertions(+), 81 deletions(-) diff --git a/management/server/account.go b/management/server/account.go index da3203852..a9781b385 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -76,6 +76,7 @@ type AccountManager interface { SaveOrAddUsers(ctx context.Context, accountID, initiatorUserID string, updates []*User, addIfNotExists bool) ([]*UserInfo, error) GetSetupKey(ctx context.Context, accountID, userID, keyID string) (*SetupKey, error) GetAccountByID(ctx context.Context, accountID string, userID string) (*Account, error) + AccountExists(ctx context.Context, accountID string) (bool, error) GetAccountIDByUserID(ctx context.Context, userID, domain string) (string, error) GetAccountIDFromToken(ctx context.Context, claims jwtclaims.AuthorizationClaims) (string, string, error) CheckUserAccessByJWTGroups(ctx context.Context, claims jwtclaims.AuthorizationClaims) error @@ -1261,6 +1262,11 @@ func (am *DefaultAccountManager) DeleteAccount(ctx context.Context, accountID, u return nil } +// AccountExists checks if an account exists. +func (am *DefaultAccountManager) AccountExists(ctx context.Context, accountID string) (bool, error) { + return am.Store.AccountExists(ctx, LockingStrengthShare, accountID) +} + // GetAccountIDByUserID retrieves the account ID based on the userID provided. // If user does have an account, it returns the user's account ID. // If the user doesn't have an account, it creates one using the provided domain. diff --git a/management/server/mock_server/account_mock.go b/management/server/mock_server/account_mock.go index b6283a7e6..ec29222a4 100644 --- a/management/server/mock_server/account_mock.go +++ b/management/server/mock_server/account_mock.go @@ -27,6 +27,7 @@ type MockAccountManager struct { CreateSetupKeyFunc func(ctx context.Context, accountId string, keyName string, keyType server.SetupKeyType, expiresIn time.Duration, autoGroups []string, usageLimit int, userID string, ephemeral bool) (*server.SetupKey, error) GetSetupKeyFunc func(ctx context.Context, accountID, userID, keyID string) (*server.SetupKey, error) + AccountExistsFunc func(ctx context.Context, accountID string) (bool, error) GetAccountIDByUserIdFunc func(ctx context.Context, userId, domain string) (string, error) GetUserFunc func(ctx context.Context, claims jwtclaims.AuthorizationClaims) (*server.User, error) ListUsersFunc func(ctx context.Context, accountID string) ([]*server.User, error) @@ -58,7 +59,7 @@ type MockAccountManager struct { UpdatePeerMetaFunc func(ctx context.Context, peerID string, meta nbpeer.PeerSystemMeta) error UpdatePeerSSHKeyFunc func(ctx context.Context, peerID string, sshKey string) error UpdatePeerFunc func(ctx context.Context, accountID, userID string, peer *nbpeer.Peer) (*nbpeer.Peer, error) - CreateRouteFunc func(ctx context.Context, accountID string, prefix netip.Prefix, networkType route.NetworkType, domains domain.List, peer string, peerGroups []string, description string, netID route.NetID, masquerade bool, metric int, groups,accessControlGroupIDs []string, enabled bool, userID string, keepRoute bool) (*route.Route, error) + CreateRouteFunc func(ctx context.Context, accountID string, prefix netip.Prefix, networkType route.NetworkType, domains domain.List, peer string, peerGroups []string, description string, netID route.NetID, masquerade bool, metric int, groups, accessControlGroupIDs []string, enabled bool, userID string, keepRoute bool) (*route.Route, error) GetRouteFunc func(ctx context.Context, accountID string, routeID route.ID, userID string) (*route.Route, error) SaveRouteFunc func(ctx context.Context, accountID string, userID string, route *route.Route) error DeleteRouteFunc func(ctx context.Context, accountID string, routeID route.ID, userID string) error @@ -194,6 +195,14 @@ func (am *MockAccountManager) CreateSetupKey( return nil, status.Errorf(codes.Unimplemented, "method CreateSetupKey is not implemented") } +// AccountExists mock implementation of AccountExists from server.AccountManager interface +func (am *MockAccountManager) AccountExists(ctx context.Context, accountID string) (bool, error) { + if am.GetAccountIDByUserIdFunc != nil { + return am.AccountExistsFunc(ctx, accountID) + } + return false, status.Errorf(codes.Unimplemented, "method AccountExists is not implemented") +} + // GetAccountIDByUserID mock implementation of GetAccountIDByUserID from server.AccountManager interface func (am *MockAccountManager) GetAccountIDByUserID(ctx context.Context, userId, domain string) (string, error) { if am.GetAccountIDByUserIdFunc != nil { @@ -444,7 +453,7 @@ func (am *MockAccountManager) UpdatePeer(ctx context.Context, accountID, userID // CreateRoute mock implementation of CreateRoute from server.AccountManager interface func (am *MockAccountManager) CreateRoute(ctx context.Context, accountID string, prefix netip.Prefix, networkType route.NetworkType, domains domain.List, peerID string, peerGroupIDs []string, description string, netID route.NetID, masquerade bool, metric int, groups, accessControlGroupID []string, enabled bool, userID string, keepRoute bool) (*route.Route, error) { if am.CreateRouteFunc != nil { - return am.CreateRouteFunc(ctx, accountID, prefix, networkType, domains, peerID, peerGroupIDs, description, netID, masquerade, metric, groups,accessControlGroupID, enabled, userID, keepRoute) + return am.CreateRouteFunc(ctx, accountID, prefix, networkType, domains, peerID, peerGroupIDs, description, netID, masquerade, metric, groups, accessControlGroupID, enabled, userID, keepRoute) } return nil, status.Errorf(codes.Unimplemented, "method CreateRoute is not implemented") } diff --git a/management/server/sql_store.go b/management/server/sql_store.go index 9e1ab27dc..d056015d8 100644 --- a/management/server/sql_store.go +++ b/management/server/sql_store.go @@ -10,7 +10,6 @@ import ( "path/filepath" "runtime" "runtime/debug" - "slices" "strings" "sync" "time" @@ -1033,84 +1032,6 @@ func (s *SqlStore) AddPeerToGroup(ctx context.Context, accountId string, peerId return nil } -// AddUserPeersToGroups adds the user's peers to specified groups in database. -func (s *SqlStore) AddUserPeersToGroups(ctx context.Context, accountID string, userID string, groupIDs []string) error { - if len(groupIDs) == 0 { - return nil - } - - var userPeerIDs []string - result := s.db.WithContext(ctx).Clauses(clause.Locking{Strength: string(LockingStrengthShare)}).Select("id"). - Where("account_id = ? AND user_id = ?", accountID, userID).Model(&nbpeer.Peer{}).Find(&userPeerIDs) - if result.Error != nil { - return status.Errorf(status.Internal, "issue finding user peers") - } - - groupsToUpdate := make([]*nbgroup.Group, 0, len(groupIDs)) - for _, gid := range groupIDs { - group, err := s.GetGroupByID(ctx, LockingStrengthShare, gid, accountID) - if err != nil { - return err - } - - groupPeers := make(map[string]struct{}) - for _, pid := range group.Peers { - groupPeers[pid] = struct{}{} - } - - for _, pid := range userPeerIDs { - groupPeers[pid] = struct{}{} - } - - group.Peers = group.Peers[:0] - for pid := range groupPeers { - group.Peers = append(group.Peers, pid) - } - - groupsToUpdate = append(groupsToUpdate, group) - } - - return s.SaveGroups(ctx, LockingStrengthUpdate, groupsToUpdate) -} - -// RemoveUserPeersFromGroups removes the user's peers from specified groups in database. -func (s *SqlStore) RemoveUserPeersFromGroups(ctx context.Context, accountID string, userID string, groupIDs []string) error { - if len(groupIDs) == 0 { - return nil - } - - var userPeerIDs []string - result := s.db.WithContext(ctx).Clauses(clause.Locking{Strength: string(LockingStrengthShare)}).Select("id"). - Where("account_id = ? AND user_id = ?", accountID, userID).Model(&nbpeer.Peer{}).Find(&userPeerIDs) - if result.Error != nil { - return status.Errorf(status.Internal, "issue finding user peers") - } - - groupsToUpdate := make([]*nbgroup.Group, 0, len(groupIDs)) - for _, gid := range groupIDs { - group, err := s.GetGroupByID(ctx, LockingStrengthShare, gid, accountID) - if err != nil { - return err - } - - if group.Name == "All" { - continue - } - - update := make([]string, 0, len(group.Peers)) - for _, pid := range group.Peers { - if !slices.Contains(userPeerIDs, pid) { - update = append(update, pid) - } - } - - group.Peers = update - groupsToUpdate = append(groupsToUpdate, group) - } - - return s.SaveGroups(ctx, LockingStrengthUpdate, groupsToUpdate) -} - // GetUserPeers retrieves peers for a user. func (s *SqlStore) GetUserPeers(ctx context.Context, lockStrength LockingStrength, accountID, userID string) ([]*nbpeer.Peer, error) { return getRecords[*nbpeer.Peer](s.db.WithContext(ctx).Where("user_id = ?", userID), lockStrength, accountID) From 5897a48e299d5553b6b375d2bc2b4df3e2dc24f1 Mon Sep 17 00:00:00 2001 From: Bethuel Mmbaga Date: Fri, 4 Oct 2024 18:55:25 +0300 Subject: [PATCH 04/12] fix wrong reference (#2695) Signed-off-by: bcmmbaga --- management/server/mock_server/account_mock.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/management/server/mock_server/account_mock.go b/management/server/mock_server/account_mock.go index ec29222a4..74557e227 100644 --- a/management/server/mock_server/account_mock.go +++ b/management/server/mock_server/account_mock.go @@ -197,7 +197,7 @@ func (am *MockAccountManager) CreateSetupKey( // AccountExists mock implementation of AccountExists from server.AccountManager interface func (am *MockAccountManager) AccountExists(ctx context.Context, accountID string) (bool, error) { - if am.GetAccountIDByUserIdFunc != nil { + if am.AccountExistsFunc != nil { return am.AccountExistsFunc(ctx, accountID) } return false, status.Errorf(codes.Unimplemented, "method AccountExists is not implemented") From f603cd92027f76c91b7d14ebb64ae6f3aa328639 Mon Sep 17 00:00:00 2001 From: Carlos Hernandez Date: Fri, 4 Oct 2024 11:15:16 -0600 Subject: [PATCH 05/12] [client] Check wginterface instead of engine ctx (#2676) Moving code to ensure wgInterface is gone right after context is cancelled/stop in the off chance that on next retry the backoff operation is permanently cancelled and interface is abandoned without destroying. --- client/internal/connect.go | 15 +++++++++------ client/internal/engine.go | 26 ++++++++++++++------------ 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/client/internal/connect.go b/client/internal/connect.go index c77f95603..74dc1f1b5 100644 --- a/client/internal/connect.go +++ b/client/internal/connect.go @@ -269,12 +269,6 @@ func (c *ConnectClient) run( checks := loginResp.GetChecks() c.engineMutex.Lock() - if c.engine != nil && c.engine.ctx.Err() != nil { - log.Info("Stopping Netbird Engine") - if err := c.engine.Stop(); err != nil { - log.Errorf("Failed to stop engine: %v", err) - } - } c.engine = NewEngineWithProbes(engineCtx, cancel, signalClient, mgmClient, relayManager, engineConfig, mobileDependency, c.statusRecorder, probes, checks) c.engineMutex.Unlock() @@ -294,6 +288,15 @@ func (c *ConnectClient) run( } <-engineCtx.Done() + c.engineMutex.Lock() + if c.engine != nil && c.engine.wgInterface != nil { + log.Infof("ensuring %s is removed, Netbird engine context cancelled", c.engine.wgInterface.Name()) + if err := c.engine.Stop(); err != nil { + log.Errorf("Failed to stop engine: %v", err) + } + c.engine = nil + } + c.engineMutex.Unlock() c.statusRecorder.ClientTeardown() backOff.Reset() diff --git a/client/internal/engine.go b/client/internal/engine.go index c51901a22..eac8ec098 100644 --- a/client/internal/engine.go +++ b/client/internal/engine.go @@ -251,6 +251,13 @@ func (e *Engine) Stop() error { } log.Info("Network monitor: stopped") + // stop/restore DNS first so dbus and friends don't complain because of a missing interface + e.stopDNSServer() + + if e.routeManager != nil { + e.routeManager.Stop() + } + err := e.removeAllPeers() if err != nil { return fmt.Errorf("failed to remove all peers: %s", err) @@ -1116,18 +1123,12 @@ func (e *Engine) close() { } } - // stop/restore DNS first so dbus and friends don't complain because of a missing interface - e.stopDNSServer() - - if e.routeManager != nil { - e.routeManager.Stop() - } - log.Debugf("removing Netbird interface %s", e.config.WgIfaceName) if e.wgInterface != nil { if err := e.wgInterface.Close(); err != nil { log.Errorf("failed closing Netbird interface %s %v", e.config.WgIfaceName, err) } + e.wgInterface = nil } if !isNil(e.sshServer) { @@ -1395,7 +1396,7 @@ func (e *Engine) startNetworkMonitor() { } // Set a new timer to debounce rapid network changes - debounceTimer = time.AfterFunc(1*time.Second, func() { + debounceTimer = time.AfterFunc(2*time.Second, func() { // This function is called after the debounce period mu.Lock() defer mu.Unlock() @@ -1426,6 +1427,11 @@ func (e *Engine) addrViaRoutes(addr netip.Addr) (bool, netip.Prefix, error) { } func (e *Engine) stopDNSServer() { + if e.dnsServer == nil { + return + } + e.dnsServer.Stop() + e.dnsServer = nil err := fmt.Errorf("DNS server stopped") nsGroupStates := e.statusRecorder.GetDNSStates() for i := range nsGroupStates { @@ -1433,10 +1439,6 @@ func (e *Engine) stopDNSServer() { nsGroupStates[i].Error = err } e.statusRecorder.UpdateDNSStates(nsGroupStates) - if e.dnsServer != nil { - e.dnsServer.Stop() - e.dnsServer = nil - } } // isChecksEqual checks if two slices of checks are equal. From dbec24b52080bd14739e9b0dd8c950e4b0edf0cb Mon Sep 17 00:00:00 2001 From: pascal-fischer <32096965+pascal-fischer@users.noreply.github.com> Date: Sun, 6 Oct 2024 17:01:13 +0200 Subject: [PATCH 06/12] [management] Remove admin check on getAccountByID (#2699) --- management/server/account.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/management/server/account.go b/management/server/account.go index a9781b385..6ee0015f8 100644 --- a/management/server/account.go +++ b/management/server/account.go @@ -20,6 +20,11 @@ import ( cacheStore "github.com/eko/gocache/v3/store" "github.com/hashicorp/go-multierror" "github.com/miekg/dns" + gocache "github.com/patrickmn/go-cache" + "github.com/rs/xid" + log "github.com/sirupsen/logrus" + "golang.org/x/exp/maps" + "github.com/netbirdio/netbird/base62" nbdns "github.com/netbirdio/netbird/dns" "github.com/netbirdio/netbird/management/domain" @@ -36,10 +41,6 @@ import ( "github.com/netbirdio/netbird/management/server/status" "github.com/netbirdio/netbird/management/server/telemetry" "github.com/netbirdio/netbird/route" - gocache "github.com/patrickmn/go-cache" - "github.com/rs/xid" - log "github.com/sirupsen/logrus" - "golang.org/x/exp/maps" ) const ( @@ -1764,7 +1765,7 @@ func (am *DefaultAccountManager) GetAccountByID(ctx context.Context, accountID s return nil, err } - if user.AccountID != accountID || (!user.HasAdminPower() && !user.IsServiceUser) { + if user.AccountID != accountID { return nil, status.Errorf(status.PermissionDenied, "the user has no permission to access account data") } From 2c1f5e46d5928a21458749251c4ee5eb96575239 Mon Sep 17 00:00:00 2001 From: Bethuel Mmbaga Date: Mon, 7 Oct 2024 19:06:26 +0300 Subject: [PATCH 07/12] [management] Validate peer ownership during login (#2704) * check peer ownership in login Signed-off-by: bcmmbaga * update error message Signed-off-by: bcmmbaga --------- Signed-off-by: bcmmbaga --- management/server/peer.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/management/server/peer.go b/management/server/peer.go index da9586734..a7d4f3b06 100644 --- a/management/server/peer.go +++ b/management/server/peer.go @@ -693,6 +693,11 @@ func (am *DefaultAccountManager) LoginPeer(ctx context.Context, login PeerLogin) updateRemotePeers := false if login.UserID != "" { + if peer.UserID != login.UserID { + log.Warnf("user mismatch when logging in peer %s: peer user %s, login user %s ", peer.ID, peer.UserID, login.UserID) + return nil, nil, nil, status.Errorf(status.Unauthenticated, "invalid user") + } + changed, err := am.handleUserPeer(ctx, peer, settings) if err != nil { return nil, nil, nil, err From 44e81073832c2d1167ef6ec284c8bb9f4c5bc3d8 Mon Sep 17 00:00:00 2001 From: Viktor Liu <17948409+lixmal@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:21:11 +0200 Subject: [PATCH 08/12] [client] Limit P2P attempts and restart on specific events (#2657) --- client/internal/peer/conn.go | 60 ++++--- client/internal/peer/conn_monitor.go | 212 +++++++++++++++++++++++++ client/internal/peer/stdnet.go | 4 +- client/internal/peer/stdnet_android.go | 4 +- client/internal/peer/worker_ice.go | 63 ++++---- 5 files changed, 285 insertions(+), 58 deletions(-) create mode 100644 client/internal/peer/conn_monitor.go diff --git a/client/internal/peer/conn.go b/client/internal/peer/conn.go index ad84bd700..0d4ad2396 100644 --- a/client/internal/peer/conn.go +++ b/client/internal/peer/conn.go @@ -32,6 +32,8 @@ const ( connPriorityRelay ConnPriority = 1 connPriorityICETurn ConnPriority = 1 connPriorityICEP2P ConnPriority = 2 + + reconnectMaxElapsedTime = 30 * time.Minute ) type WgConfig struct { @@ -83,6 +85,7 @@ type Conn struct { wgProxyICE wgproxy.Proxy wgProxyRelay wgproxy.Proxy signaler *Signaler + iFaceDiscover stdnet.ExternalIFaceDiscover relayManager *relayClient.Manager allowedIPsIP string handshaker *Handshaker @@ -108,6 +111,8 @@ type Conn struct { // for reconnection operations iCEDisconnected chan bool relayDisconnected chan bool + connMonitor *ConnMonitor + reconnectCh <-chan struct{} } // NewConn creates a new not opened Conn to the remote peer. @@ -123,21 +128,31 @@ func NewConn(engineCtx context.Context, config ConnConfig, statusRecorder *Statu connLog := log.WithField("peer", config.Key) var conn = &Conn{ - log: connLog, - ctx: ctx, - ctxCancel: ctxCancel, - config: config, - statusRecorder: statusRecorder, - wgProxyFactory: wgProxyFactory, - signaler: signaler, - relayManager: relayManager, - allowedIPsIP: allowedIPsIP.String(), - statusRelay: NewAtomicConnStatus(), - statusICE: NewAtomicConnStatus(), + log: connLog, + ctx: ctx, + ctxCancel: ctxCancel, + config: config, + statusRecorder: statusRecorder, + wgProxyFactory: wgProxyFactory, + signaler: signaler, + iFaceDiscover: iFaceDiscover, + relayManager: relayManager, + allowedIPsIP: allowedIPsIP.String(), + statusRelay: NewAtomicConnStatus(), + statusICE: NewAtomicConnStatus(), + iCEDisconnected: make(chan bool, 1), relayDisconnected: make(chan bool, 1), } + conn.connMonitor, conn.reconnectCh = NewConnMonitor( + signaler, + iFaceDiscover, + config, + conn.relayDisconnected, + conn.iCEDisconnected, + ) + rFns := WorkerRelayCallbacks{ OnConnReady: conn.relayConnectionIsReady, OnDisconnected: conn.onWorkerRelayStateDisconnected, @@ -200,6 +215,8 @@ func (conn *Conn) startHandshakeAndReconnect() { conn.log.Errorf("failed to send initial offer: %v", err) } + go conn.connMonitor.Start(conn.ctx) + if conn.workerRelay.IsController() { conn.reconnectLoopWithRetry() } else { @@ -309,12 +326,14 @@ func (conn *Conn) reconnectLoopWithRetry() { // With it, we can decrease to send necessary offer select { case <-conn.ctx.Done(): + return case <-time.After(3 * time.Second): } ticker := conn.prepareExponentTicker() defer ticker.Stop() time.Sleep(1 * time.Second) + for { select { case t := <-ticker.C: @@ -342,20 +361,11 @@ func (conn *Conn) reconnectLoopWithRetry() { if err != nil { conn.log.Errorf("failed to do handshake: %v", err) } - case changed := <-conn.relayDisconnected: - if !changed { - continue - } - conn.log.Debugf("Relay state changed, reset reconnect timer") - ticker.Stop() - ticker = conn.prepareExponentTicker() - case changed := <-conn.iCEDisconnected: - if !changed { - continue - } - conn.log.Debugf("ICE state changed, reset reconnect timer") + + case <-conn.reconnectCh: ticker.Stop() ticker = conn.prepareExponentTicker() + case <-conn.ctx.Done(): conn.log.Debugf("context is done, stop reconnect loop") return @@ -366,10 +376,10 @@ func (conn *Conn) reconnectLoopWithRetry() { func (conn *Conn) prepareExponentTicker() *backoff.Ticker { bo := backoff.WithContext(&backoff.ExponentialBackOff{ InitialInterval: 800 * time.Millisecond, - RandomizationFactor: 0.01, + RandomizationFactor: 0.1, Multiplier: 2, MaxInterval: conn.config.Timeout, - MaxElapsedTime: 0, + MaxElapsedTime: reconnectMaxElapsedTime, Stop: backoff.Stop, Clock: backoff.SystemClock, }, conn.ctx) diff --git a/client/internal/peer/conn_monitor.go b/client/internal/peer/conn_monitor.go new file mode 100644 index 000000000..75722c990 --- /dev/null +++ b/client/internal/peer/conn_monitor.go @@ -0,0 +1,212 @@ +package peer + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/pion/ice/v3" + log "github.com/sirupsen/logrus" + + "github.com/netbirdio/netbird/client/internal/stdnet" +) + +const ( + signalerMonitorPeriod = 5 * time.Second + candidatesMonitorPeriod = 5 * time.Minute + candidateGatheringTimeout = 5 * time.Second +) + +type ConnMonitor struct { + signaler *Signaler + iFaceDiscover stdnet.ExternalIFaceDiscover + config ConnConfig + relayDisconnected chan bool + iCEDisconnected chan bool + reconnectCh chan struct{} + currentCandidates []ice.Candidate + candidatesMu sync.Mutex +} + +func NewConnMonitor(signaler *Signaler, iFaceDiscover stdnet.ExternalIFaceDiscover, config ConnConfig, relayDisconnected, iCEDisconnected chan bool) (*ConnMonitor, <-chan struct{}) { + reconnectCh := make(chan struct{}, 1) + cm := &ConnMonitor{ + signaler: signaler, + iFaceDiscover: iFaceDiscover, + config: config, + relayDisconnected: relayDisconnected, + iCEDisconnected: iCEDisconnected, + reconnectCh: reconnectCh, + } + return cm, reconnectCh +} + +func (cm *ConnMonitor) Start(ctx context.Context) { + signalerReady := make(chan struct{}, 1) + go cm.monitorSignalerReady(ctx, signalerReady) + + localCandidatesChanged := make(chan struct{}, 1) + go cm.monitorLocalCandidatesChanged(ctx, localCandidatesChanged) + + for { + select { + case changed := <-cm.relayDisconnected: + if !changed { + continue + } + log.Debugf("Relay state changed, triggering reconnect") + cm.triggerReconnect() + + case changed := <-cm.iCEDisconnected: + if !changed { + continue + } + log.Debugf("ICE state changed, triggering reconnect") + cm.triggerReconnect() + + case <-signalerReady: + log.Debugf("Signaler became ready, triggering reconnect") + cm.triggerReconnect() + + case <-localCandidatesChanged: + log.Debugf("Local candidates changed, triggering reconnect") + cm.triggerReconnect() + + case <-ctx.Done(): + return + } + } +} + +func (cm *ConnMonitor) monitorSignalerReady(ctx context.Context, signalerReady chan<- struct{}) { + if cm.signaler == nil { + return + } + + ticker := time.NewTicker(signalerMonitorPeriod) + defer ticker.Stop() + + lastReady := true + for { + select { + case <-ticker.C: + currentReady := cm.signaler.Ready() + if !lastReady && currentReady { + select { + case signalerReady <- struct{}{}: + default: + } + } + lastReady = currentReady + case <-ctx.Done(): + return + } + } +} + +func (cm *ConnMonitor) monitorLocalCandidatesChanged(ctx context.Context, localCandidatesChanged chan<- struct{}) { + ufrag, pwd, err := generateICECredentials() + if err != nil { + log.Warnf("Failed to generate ICE credentials: %v", err) + return + } + + ticker := time.NewTicker(candidatesMonitorPeriod) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if err := cm.handleCandidateTick(ctx, localCandidatesChanged, ufrag, pwd); err != nil { + log.Warnf("Failed to handle candidate tick: %v", err) + } + case <-ctx.Done(): + return + } + } +} + +func (cm *ConnMonitor) handleCandidateTick(ctx context.Context, localCandidatesChanged chan<- struct{}, ufrag string, pwd string) error { + log.Debugf("Gathering ICE candidates") + + transportNet, err := newStdNet(cm.iFaceDiscover, cm.config.ICEConfig.InterfaceBlackList) + if err != nil { + log.Errorf("failed to create pion's stdnet: %s", err) + } + + agent, err := newAgent(cm.config, transportNet, candidateTypesP2P(), ufrag, pwd) + if err != nil { + return fmt.Errorf("create ICE agent: %w", err) + } + defer func() { + if err := agent.Close(); err != nil { + log.Warnf("Failed to close ICE agent: %v", err) + } + }() + + gatherDone := make(chan struct{}) + err = agent.OnCandidate(func(c ice.Candidate) { + log.Tracef("Got candidate: %v", c) + if c == nil { + close(gatherDone) + } + }) + if err != nil { + return fmt.Errorf("set ICE candidate handler: %w", err) + } + + if err := agent.GatherCandidates(); err != nil { + return fmt.Errorf("gather ICE candidates: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, candidateGatheringTimeout) + defer cancel() + + select { + case <-ctx.Done(): + return fmt.Errorf("wait for gathering: %w", ctx.Err()) + case <-gatherDone: + } + + candidates, err := agent.GetLocalCandidates() + if err != nil { + return fmt.Errorf("get local candidates: %w", err) + } + log.Tracef("Got candidates: %v", candidates) + + if changed := cm.updateCandidates(candidates); changed { + select { + case localCandidatesChanged <- struct{}{}: + default: + } + } + + return nil +} + +func (cm *ConnMonitor) updateCandidates(newCandidates []ice.Candidate) bool { + cm.candidatesMu.Lock() + defer cm.candidatesMu.Unlock() + + if len(cm.currentCandidates) != len(newCandidates) { + cm.currentCandidates = newCandidates + return true + } + + for i, candidate := range cm.currentCandidates { + if candidate.Address() != newCandidates[i].Address() { + cm.currentCandidates = newCandidates + return true + } + } + + return false +} + +func (cm *ConnMonitor) triggerReconnect() { + select { + case cm.reconnectCh <- struct{}{}: + default: + } +} diff --git a/client/internal/peer/stdnet.go b/client/internal/peer/stdnet.go index ae31ebbf0..96d211dbc 100644 --- a/client/internal/peer/stdnet.go +++ b/client/internal/peer/stdnet.go @@ -6,6 +6,6 @@ import ( "github.com/netbirdio/netbird/client/internal/stdnet" ) -func (w *WorkerICE) newStdNet() (*stdnet.Net, error) { - return stdnet.NewNet(w.config.ICEConfig.InterfaceBlackList) +func newStdNet(_ stdnet.ExternalIFaceDiscover, ifaceBlacklist []string) (*stdnet.Net, error) { + return stdnet.NewNet(ifaceBlacklist) } diff --git a/client/internal/peer/stdnet_android.go b/client/internal/peer/stdnet_android.go index b411405bb..a39a03b1c 100644 --- a/client/internal/peer/stdnet_android.go +++ b/client/internal/peer/stdnet_android.go @@ -2,6 +2,6 @@ package peer import "github.com/netbirdio/netbird/client/internal/stdnet" -func (w *WorkerICE) newStdNet() (*stdnet.Net, error) { - return stdnet.NewNetWithDiscover(w.iFaceDiscover, w.config.ICEConfig.InterfaceBlackList) +func newStdNet(iFaceDiscover stdnet.ExternalIFaceDiscover, ifaceBlacklist []string) (*stdnet.Net, error) { + return stdnet.NewNetWithDiscover(iFaceDiscover, ifaceBlacklist) } diff --git a/client/internal/peer/worker_ice.go b/client/internal/peer/worker_ice.go index c4e9d1950..c86c1858f 100644 --- a/client/internal/peer/worker_ice.go +++ b/client/internal/peer/worker_ice.go @@ -233,41 +233,16 @@ func (w *WorkerICE) Close() { } func (w *WorkerICE) reCreateAgent(agentCancel context.CancelFunc, relaySupport []ice.CandidateType) (*ice.Agent, error) { - transportNet, err := w.newStdNet() + transportNet, err := newStdNet(w.iFaceDiscover, w.config.ICEConfig.InterfaceBlackList) if err != nil { w.log.Errorf("failed to create pion's stdnet: %s", err) } - iceKeepAlive := iceKeepAlive() - iceDisconnectedTimeout := iceDisconnectedTimeout() - iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait() - - agentConfig := &ice.AgentConfig{ - MulticastDNSMode: ice.MulticastDNSModeDisabled, - NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, - Urls: w.config.ICEConfig.StunTurn.Load().([]*stun.URI), - CandidateTypes: relaySupport, - InterfaceFilter: stdnet.InterfaceFilter(w.config.ICEConfig.InterfaceBlackList), - UDPMux: w.config.ICEConfig.UDPMux, - UDPMuxSrflx: w.config.ICEConfig.UDPMuxSrflx, - NAT1To1IPs: w.config.ICEConfig.NATExternalIPs, - Net: transportNet, - FailedTimeout: &failedTimeout, - DisconnectedTimeout: &iceDisconnectedTimeout, - KeepaliveInterval: &iceKeepAlive, - RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait, - LocalUfrag: w.localUfrag, - LocalPwd: w.localPwd, - } - - if w.config.ICEConfig.DisableIPv6Discovery { - agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4} - } - w.sentExtraSrflx = false - agent, err := ice.NewAgent(agentConfig) + + agent, err := newAgent(w.config, transportNet, relaySupport, w.localUfrag, w.localPwd) if err != nil { - return nil, err + return nil, fmt.Errorf("create agent: %w", err) } err = agent.OnCandidate(w.onICECandidate) @@ -390,6 +365,36 @@ func (w *WorkerICE) turnAgentDial(ctx context.Context, remoteOfferAnswer *OfferA } } +func newAgent(config ConnConfig, transportNet *stdnet.Net, candidateTypes []ice.CandidateType, ufrag string, pwd string) (*ice.Agent, error) { + iceKeepAlive := iceKeepAlive() + iceDisconnectedTimeout := iceDisconnectedTimeout() + iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait() + + agentConfig := &ice.AgentConfig{ + MulticastDNSMode: ice.MulticastDNSModeDisabled, + NetworkTypes: []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeUDP6}, + Urls: config.ICEConfig.StunTurn.Load().([]*stun.URI), + CandidateTypes: candidateTypes, + InterfaceFilter: stdnet.InterfaceFilter(config.ICEConfig.InterfaceBlackList), + UDPMux: config.ICEConfig.UDPMux, + UDPMuxSrflx: config.ICEConfig.UDPMuxSrflx, + NAT1To1IPs: config.ICEConfig.NATExternalIPs, + Net: transportNet, + FailedTimeout: &failedTimeout, + DisconnectedTimeout: &iceDisconnectedTimeout, + KeepaliveInterval: &iceKeepAlive, + RelayAcceptanceMinWait: &iceRelayAcceptanceMinWait, + LocalUfrag: ufrag, + LocalPwd: pwd, + } + + if config.ICEConfig.DisableIPv6Discovery { + agentConfig.NetworkTypes = []ice.NetworkType{ice.NetworkTypeUDP4} + } + + return ice.NewAgent(agentConfig) +} + func extraSrflxCandidate(candidate ice.Candidate) (*ice.CandidateServerReflexive, error) { relatedAdd := candidate.RelatedAddress() return ice.NewCandidateServerReflexive(&ice.CandidateServerReflexiveConfig{ From d4ef84fe6e02e932fdfa24be2bcf2416634f832b Mon Sep 17 00:00:00 2001 From: pascal-fischer <32096965+pascal-fischer@users.noreply.github.com> Date: Wed, 9 Oct 2024 14:33:58 +0200 Subject: [PATCH 09/12] [management] Propagate error in store errors (#2709) --- management/server/sql_store.go | 46 +++++++++++++++---------------- management/server/status/error.go | 8 ++++-- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/management/server/sql_store.go b/management/server/sql_store.go index d056015d8..67df29ef0 100644 --- a/management/server/sql_store.go +++ b/management/server/sql_store.go @@ -431,7 +431,7 @@ func (s *SqlStore) GetAccountIDByPrivateDomain(ctx context.Context, lockStrength return "", status.Errorf(status.NotFound, "account not found: provided domain is not registered or is not private") } log.WithContext(ctx).Errorf("error when getting account from the store: %s", result.Error) - return "", status.Errorf(status.Internal, "issue getting account from store") + return "", status.NewGetAccountFromStoreError(result.Error) } return accountID, nil @@ -444,7 +444,7 @@ func (s *SqlStore) GetAccountBySetupKey(ctx context.Context, setupKey string) (* if errors.Is(result.Error, gorm.ErrRecordNotFound) { return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") } - return nil, status.NewSetupKeyNotFoundError() + return nil, status.NewSetupKeyNotFoundError(result.Error) } if key.AccountID == "" { @@ -462,7 +462,7 @@ func (s *SqlStore) GetTokenIDByHashedToken(ctx context.Context, hashedToken stri return "", status.Errorf(status.NotFound, "account not found: index lookup failed") } log.WithContext(ctx).Errorf("error when getting token from the store: %s", result.Error) - return "", status.Errorf(status.Internal, "issue getting account from store") + return "", status.NewGetAccountFromStoreError(result.Error) } return token.ID, nil @@ -476,7 +476,7 @@ func (s *SqlStore) GetUserByTokenID(ctx context.Context, tokenID string) (*User, return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") } log.WithContext(ctx).Errorf("error when getting token from the store: %s", result.Error) - return nil, status.Errorf(status.Internal, "issue getting account from store") + return nil, status.NewGetAccountFromStoreError(result.Error) } if token.UserID == "" { @@ -560,7 +560,7 @@ func (s *SqlStore) GetAccount(ctx context.Context, accountID string) (*Account, if errors.Is(result.Error, gorm.ErrRecordNotFound) { return nil, status.NewAccountNotFoundError(accountID) } - return nil, status.Errorf(status.Internal, "issue getting account from store") + return nil, status.NewGetAccountFromStoreError(result.Error) } // we have to manually preload policy rules as it seems that gorm preloading doesn't do it for us @@ -623,7 +623,7 @@ func (s *SqlStore) GetAccountByUser(ctx context.Context, userID string) (*Accoun if errors.Is(result.Error, gorm.ErrRecordNotFound) { return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") } - return nil, status.Errorf(status.Internal, "issue getting account from store") + return nil, status.NewGetAccountFromStoreError(result.Error) } if user.AccountID == "" { @@ -640,7 +640,7 @@ func (s *SqlStore) GetAccountByPeerID(ctx context.Context, peerID string) (*Acco if errors.Is(result.Error, gorm.ErrRecordNotFound) { return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") } - return nil, status.Errorf(status.Internal, "issue getting account from store") + return nil, status.NewGetAccountFromStoreError(result.Error) } if peer.AccountID == "" { @@ -658,7 +658,7 @@ func (s *SqlStore) GetAccountByPeerPubKey(ctx context.Context, peerKey string) ( if errors.Is(result.Error, gorm.ErrRecordNotFound) { return nil, status.Errorf(status.NotFound, "account not found: index lookup failed") } - return nil, status.Errorf(status.Internal, "issue getting account from store") + return nil, status.NewGetAccountFromStoreError(result.Error) } if peer.AccountID == "" { @@ -676,7 +676,7 @@ func (s *SqlStore) GetAccountIDByPeerPubKey(ctx context.Context, peerKey string) if errors.Is(result.Error, gorm.ErrRecordNotFound) { return "", status.Errorf(status.NotFound, "account not found: index lookup failed") } - return "", status.Errorf(status.Internal, "issue getting account from store") + return "", status.NewGetAccountFromStoreError(result.Error) } return accountID, nil @@ -689,7 +689,7 @@ func (s *SqlStore) GetAccountIDByUserID(userID string) (string, error) { if errors.Is(result.Error, gorm.ErrRecordNotFound) { return "", status.Errorf(status.NotFound, "account not found: index lookup failed") } - return "", status.Errorf(status.Internal, "issue getting account from store") + return "", status.NewGetAccountFromStoreError(result.Error) } return accountID, nil @@ -702,7 +702,7 @@ func (s *SqlStore) GetAccountIDBySetupKey(ctx context.Context, setupKey string) if errors.Is(result.Error, gorm.ErrRecordNotFound) { return "", status.Errorf(status.NotFound, "account not found: index lookup failed") } - return "", status.NewSetupKeyNotFoundError() + return "", status.NewSetupKeyNotFoundError(result.Error) } if accountID == "" { @@ -723,7 +723,7 @@ func (s *SqlStore) GetTakenIPs(ctx context.Context, lockStrength LockingStrength if errors.Is(result.Error, gorm.ErrRecordNotFound) { return nil, status.Errorf(status.NotFound, "no peers found for the account") } - return nil, status.Errorf(status.Internal, "issue getting IPs from store") + return nil, status.Errorf(status.Internal, "issue getting IPs from store: %s", result.Error) } // Convert the JSON strings to net.IP objects @@ -751,7 +751,7 @@ func (s *SqlStore) GetPeerLabelsInAccount(ctx context.Context, lockStrength Lock return nil, status.Errorf(status.NotFound, "no peers found for the account") } log.WithContext(ctx).Errorf("error when getting dns labels from the store: %s", result.Error) - return nil, status.Errorf(status.Internal, "issue getting dns labels from store") + return nil, status.Errorf(status.Internal, "issue getting dns labels from store: %s", result.Error) } return labels, nil @@ -764,7 +764,7 @@ func (s *SqlStore) GetAccountNetwork(ctx context.Context, lockStrength LockingSt if errors.Is(err, gorm.ErrRecordNotFound) { return nil, status.NewAccountNotFoundError(accountID) } - return nil, status.Errorf(status.Internal, "issue getting network from store") + return nil, status.Errorf(status.Internal, "issue getting network from store: %s", err) } return accountNetwork.Network, nil } @@ -776,7 +776,7 @@ func (s *SqlStore) GetPeerByPeerPubKey(ctx context.Context, lockStrength Locking if errors.Is(result.Error, gorm.ErrRecordNotFound) { return nil, status.Errorf(status.NotFound, "peer not found") } - return nil, status.Errorf(status.Internal, "issue getting peer from store") + return nil, status.Errorf(status.Internal, "issue getting peer from store: %s", result.Error) } return &peer, nil @@ -788,7 +788,7 @@ func (s *SqlStore) GetAccountSettings(ctx context.Context, lockStrength LockingS if errors.Is(err, gorm.ErrRecordNotFound) { return nil, status.Errorf(status.NotFound, "settings not found") } - return nil, status.Errorf(status.Internal, "issue getting settings from store") + return nil, status.Errorf(status.Internal, "issue getting settings from store: %s", err) } return accountSettings.Settings, nil } @@ -956,7 +956,7 @@ func (s *SqlStore) GetSetupKeyBySecret(ctx context.Context, lockStrength Locking if errors.Is(result.Error, gorm.ErrRecordNotFound) { return nil, status.Errorf(status.NotFound, "setup key not found") } - return nil, status.NewSetupKeyNotFoundError() + return nil, status.NewSetupKeyNotFoundError(result.Error) } return &setupKey, nil } @@ -988,7 +988,7 @@ func (s *SqlStore) AddPeerToAllGroup(ctx context.Context, accountID string, peer if errors.Is(result.Error, gorm.ErrRecordNotFound) { return status.Errorf(status.NotFound, "group 'All' not found for account") } - return status.Errorf(status.Internal, "issue finding group 'All'") + return status.Errorf(status.Internal, "issue finding group 'All': %s", result.Error) } for _, existingPeerID := range group.Peers { @@ -1000,7 +1000,7 @@ func (s *SqlStore) AddPeerToAllGroup(ctx context.Context, accountID string, peer group.Peers = append(group.Peers, peerID) if err := s.db.Save(&group).Error; err != nil { - return status.Errorf(status.Internal, "issue updating group 'All'") + return status.Errorf(status.Internal, "issue updating group 'All': %s", err) } return nil @@ -1014,7 +1014,7 @@ func (s *SqlStore) AddPeerToGroup(ctx context.Context, accountId string, peerId if errors.Is(result.Error, gorm.ErrRecordNotFound) { return status.Errorf(status.NotFound, "group not found for account") } - return status.Errorf(status.Internal, "issue finding group") + return status.Errorf(status.Internal, "issue finding group: %s", result.Error) } for _, existingPeerID := range group.Peers { @@ -1026,7 +1026,7 @@ func (s *SqlStore) AddPeerToGroup(ctx context.Context, accountId string, peerId group.Peers = append(group.Peers, peerId) if err := s.db.Save(&group).Error; err != nil { - return status.Errorf(status.Internal, "issue updating group") + return status.Errorf(status.Internal, "issue updating group: %s", err) } return nil @@ -1039,7 +1039,7 @@ func (s *SqlStore) GetUserPeers(ctx context.Context, lockStrength LockingStrengt func (s *SqlStore) AddPeerToAccount(ctx context.Context, peer *nbpeer.Peer) error { if err := s.db.WithContext(ctx).Create(peer).Error; err != nil { - return status.Errorf(status.Internal, "issue adding peer to account") + return status.Errorf(status.Internal, "issue adding peer to account: %s", err) } return nil @@ -1048,7 +1048,7 @@ func (s *SqlStore) AddPeerToAccount(ctx context.Context, peer *nbpeer.Peer) erro func (s *SqlStore) IncrementNetworkSerial(ctx context.Context, accountId string) error { result := s.db.WithContext(ctx).Model(&Account{}).Where(idQueryCondition, accountId).Update("network_serial", gorm.Expr("network_serial + 1")) if result.Error != nil { - return status.Errorf(status.Internal, "issue incrementing network serial count") + return status.Errorf(status.Internal, "issue incrementing network serial count: %s", result.Error) } return nil } diff --git a/management/server/status/error.go b/management/server/status/error.go index d7fde35b9..29d185216 100644 --- a/management/server/status/error.go +++ b/management/server/status/error.go @@ -102,8 +102,12 @@ func NewPeerLoginExpiredError() error { } // NewSetupKeyNotFoundError creates a new Error with NotFound type for a missing setup key -func NewSetupKeyNotFoundError() error { - return Errorf(NotFound, "setup key not found") +func NewSetupKeyNotFoundError(err error) error { + return Errorf(NotFound, "setup key not found: %s", err) +} + +func NewGetAccountFromStoreError(err error) error { + return Errorf(Internal, "issue getting account from store: %s", err) } // NewGetUserFromStoreError creates a new Error with Internal type for an issue getting user from store From b1eda43f4b748ee9940e9a2399f200b60e55f076 Mon Sep 17 00:00:00 2001 From: Misha Bragin Date: Wed, 9 Oct 2024 13:56:25 +0100 Subject: [PATCH 10/12] Add Link to the Lawrence Systems video (#2711) --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index aa3ec41e5..270c9ad87 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ ![netbird_2](https://github.com/netbirdio/netbird/assets/700848/46bc3b73-508d-4a0e-bb9a-f465d68646ab) +### NetBird on Lawrence Systems (Video) +[![Watch the video](https://img.youtube.com/vi/Kwrff6h0rEw/0.jpg)](https://www.youtube.com/watch?v=Kwrff6h0rEw) ### Key features @@ -62,6 +64,7 @@ | | |
  • - \[x] [Quantum-resistance with Rosenpass](https://netbird.io/knowledge-hub/the-first-quantum-resistant-mesh-vpn)
| |
  • - \[x] OpenWRT
| | | |
  • - \[x] [Periodic re-authentication](https://docs.netbird.io/how-to/enforce-periodic-user-authentication)
    • | |
    • - \[x] [Serverless](https://docs.netbird.io/how-to/netbird-on-faas)
    | | | | | |
    • - \[x] Docker
    | + ### Quickstart with NetBird Cloud - Download and install NetBird at [https://app.netbird.io/install](https://app.netbird.io/install) From b79c1d64cc10871e70c94ee5f47fdf2b4773f5a7 Mon Sep 17 00:00:00 2001 From: pascal-fischer <32096965+pascal-fischer@users.noreply.github.com> Date: Wed, 9 Oct 2024 20:17:25 +0200 Subject: [PATCH 11/12] [management] Make max open db conns configurable (#2713) --- .github/workflows/golang-test-linux.yml | 2 +- .github/workflows/release.yml | 2 +- management/server/sql_store.go | 11 +++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/golang-test-linux.yml b/.github/workflows/golang-test-linux.yml index 524f35f6f..d6adcb27a 100644 --- a/.github/workflows/golang-test-linux.yml +++ b/.github/workflows/golang-test-linux.yml @@ -16,7 +16,7 @@ jobs: matrix: arch: [ '386','amd64' ] store: [ 'sqlite', 'postgres'] - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Install Go uses: actions/setup-go@v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7af6d3e4d..b2e2437e6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ concurrency: jobs: release: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 env: flags: "" steps: diff --git a/management/server/sql_store.go b/management/server/sql_store.go index 67df29ef0..fe4dcafdb 100644 --- a/management/server/sql_store.go +++ b/management/server/sql_store.go @@ -10,6 +10,7 @@ import ( "path/filepath" "runtime" "runtime/debug" + "strconv" "strings" "sync" "time" @@ -63,8 +64,14 @@ func NewSqlStore(ctx context.Context, db *gorm.DB, storeEngine StoreEngine, metr if err != nil { return nil, err } - conns := runtime.NumCPU() - sql.SetMaxOpenConns(conns) // TODO: make it configurable + + conns, err := strconv.Atoi(os.Getenv("NB_SQL_MAX_OPEN_CONNS")) + if err != nil { + conns = runtime.NumCPU() + } + sql.SetMaxOpenConns(conns) + + log.Infof("Set max open db connections to %d", conns) if err := migrate(ctx, db); err != nil { return nil, fmt.Errorf("migrate: %w", err) From 6ce09bca1680c50012cd9467317290808b620808 Mon Sep 17 00:00:00 2001 From: Maycon Santos Date: Wed, 9 Oct 2024 20:46:23 +0200 Subject: [PATCH 12/12] Add support to envsub go management configurations (#2708) This change allows users to reference environment variables using Go template format, like {{ .EnvName }} Moved the previous file test code to file_suite_test.go. --- management/cmd/management.go | 2 +- util/file.go | 53 +++++++ util/file_suite_test.go | 126 +++++++++++++++ util/file_test.go | 288 ++++++++++++++++++++++------------- 4 files changed, 360 insertions(+), 109 deletions(-) create mode 100644 util/file_suite_test.go diff --git a/management/cmd/management.go b/management/cmd/management.go index 78b1a8d63..719d1a78c 100644 --- a/management/cmd/management.go +++ b/management/cmd/management.go @@ -475,7 +475,7 @@ func handlerFunc(gRPCHandler *grpc.Server, httpHandler http.Handler) http.Handle func loadMgmtConfig(ctx context.Context, mgmtConfigPath string) (*server.Config, error) { loadedConfig := &server.Config{} - _, err := util.ReadJson(mgmtConfigPath, loadedConfig) + _, err := util.ReadJsonWithEnvSub(mgmtConfigPath, loadedConfig) if err != nil { return nil, err } diff --git a/util/file.go b/util/file.go index 8355488c9..ecaecd222 100644 --- a/util/file.go +++ b/util/file.go @@ -1,11 +1,15 @@ package util import ( + "bytes" "context" "encoding/json" + "fmt" "io" "os" "path/filepath" + "strings" + "text/template" log "github.com/sirupsen/logrus" ) @@ -160,6 +164,55 @@ func ReadJson(file string, res interface{}) (interface{}, error) { return res, nil } +// ReadJsonWithEnvSub reads JSON config file and maps to a provided interface with environment variable substitution +func ReadJsonWithEnvSub(file string, res interface{}) (interface{}, error) { + envVars := getEnvMap() + + f, err := os.Open(file) + if err != nil { + return nil, err + } + defer f.Close() + + bs, err := io.ReadAll(f) + if err != nil { + return nil, err + } + + t, err := template.New("").Parse(string(bs)) + if err != nil { + return nil, fmt.Errorf("error parsing template: %v", err) + } + + var output bytes.Buffer + // Execute the template, substituting environment variables + err = t.Execute(&output, envVars) + if err != nil { + return nil, fmt.Errorf("error executing template: %v", err) + } + + err = json.Unmarshal(output.Bytes(), &res) + if err != nil { + return nil, fmt.Errorf("failed parsing Json file after template was executed, err: %v", err) + } + + return res, nil +} + +// getEnvMap Convert the output of os.Environ() to a map +func getEnvMap() map[string]string { + envMap := make(map[string]string) + + for _, env := range os.Environ() { + parts := strings.SplitN(env, "=", 2) + if len(parts) == 2 { + envMap[parts[0]] = parts[1] + } + } + + return envMap +} + // CopyFileContents copies contents of the given src file to the dst file func CopyFileContents(src, dst string) (err error) { in, err := os.Open(src) diff --git a/util/file_suite_test.go b/util/file_suite_test.go new file mode 100644 index 000000000..3de7db49b --- /dev/null +++ b/util/file_suite_test.go @@ -0,0 +1,126 @@ +package util_test + +import ( + "crypto/md5" + "encoding/hex" + "io" + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/netbirdio/netbird/util" +) + +var _ = Describe("Client", func() { + + var ( + tmpDir string + ) + + type TestConfig struct { + SomeMap map[string]string + SomeArray []string + SomeField int + } + + BeforeEach(func() { + var err error + tmpDir, err = os.MkdirTemp("", "wiretrustee_util_test_tmp_*") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + err := os.RemoveAll(tmpDir) + Expect(err).NotTo(HaveOccurred()) + }) + + Describe("Config", func() { + Context("in JSON format", func() { + It("should be written and read successfully", func() { + + m := make(map[string]string) + m["key1"] = "value1" + m["key2"] = "value2" + + arr := []string{"value1", "value2"} + + written := &TestConfig{ + SomeMap: m, + SomeArray: arr, + SomeField: 99, + } + + err := util.WriteJson(tmpDir+"/testconfig.json", written) + Expect(err).NotTo(HaveOccurred()) + + read, err := util.ReadJson(tmpDir+"/testconfig.json", &TestConfig{}) + Expect(err).NotTo(HaveOccurred()) + Expect(read).NotTo(BeNil()) + Expect(read.(*TestConfig).SomeMap["key1"]).To(BeEquivalentTo(written.SomeMap["key1"])) + Expect(read.(*TestConfig).SomeMap["key2"]).To(BeEquivalentTo(written.SomeMap["key2"])) + Expect(read.(*TestConfig).SomeArray).To(ContainElements(arr)) + Expect(read.(*TestConfig).SomeField).To(BeEquivalentTo(written.SomeField)) + + }) + }) + }) + + Describe("Copying file contents", func() { + Context("from one file to another", func() { + It("should be successful", func() { + + src := tmpDir + "/copytest_src" + dst := tmpDir + "/copytest_dst" + + err := util.WriteJson(src, []string{"1", "2", "3"}) + Expect(err).NotTo(HaveOccurred()) + + err = util.CopyFileContents(src, dst) + Expect(err).NotTo(HaveOccurred()) + + hashSrc := md5.New() + hashDst := md5.New() + + srcFile, err := os.Open(src) + Expect(err).NotTo(HaveOccurred()) + + dstFile, err := os.Open(dst) + Expect(err).NotTo(HaveOccurred()) + + _, err = io.Copy(hashSrc, srcFile) + Expect(err).NotTo(HaveOccurred()) + + _, err = io.Copy(hashDst, dstFile) + Expect(err).NotTo(HaveOccurred()) + + err = srcFile.Close() + Expect(err).NotTo(HaveOccurred()) + + err = dstFile.Close() + Expect(err).NotTo(HaveOccurred()) + + Expect(hex.EncodeToString(hashSrc.Sum(nil)[:16])).To(BeEquivalentTo(hex.EncodeToString(hashDst.Sum(nil)[:16]))) + }) + }) + }) + + Describe("Handle config file without full path", func() { + Context("config file handling", func() { + It("should be successful", func() { + written := &TestConfig{ + SomeField: 123, + } + cfgFile := "test_cfg.json" + defer os.Remove(cfgFile) + + err := util.WriteJson(cfgFile, written) + Expect(err).NotTo(HaveOccurred()) + + read, err := util.ReadJson(cfgFile, &TestConfig{}) + Expect(err).NotTo(HaveOccurred()) + Expect(read).NotTo(BeNil()) + }) + }) + }) +}) diff --git a/util/file_test.go b/util/file_test.go index 3de7db49b..1330e738e 100644 --- a/util/file_test.go +++ b/util/file_test.go @@ -1,126 +1,198 @@ -package util_test +package util import ( - "crypto/md5" - "encoding/hex" - "io" "os" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/netbirdio/netbird/util" + "reflect" + "strings" + "testing" ) -var _ = Describe("Client", func() { - - var ( - tmpDir string - ) - - type TestConfig struct { - SomeMap map[string]string - SomeArray []string - SomeField int +func TestReadJsonWithEnvSub(t *testing.T) { + type Config struct { + CertFile string `json:"CertFile"` + Credentials string `json:"Credentials"` + NestedOption struct { + URL string `json:"URL"` + } `json:"NestedOption"` } - BeforeEach(func() { - var err error - tmpDir, err = os.MkdirTemp("", "wiretrustee_util_test_tmp_*") - Expect(err).NotTo(HaveOccurred()) - }) + type testCase struct { + name string + envVars map[string]string + jsonTemplate string + expectedResult Config + expectError bool + errorContains string + } - AfterEach(func() { - err := os.RemoveAll(tmpDir) - Expect(err).NotTo(HaveOccurred()) - }) + tests := []testCase{ + { + name: "All environment variables set", + envVars: map[string]string{ + "CERT_FILE": "/etc/certs/env_cert.crt", + "CREDENTIALS": "env_credentials", + "URL": "https://env.testing.com", + }, + jsonTemplate: `{ + "CertFile": "{{ .CERT_FILE }}", + "Credentials": "{{ .CREDENTIALS }}", + "NestedOption": { + "URL": "{{ .URL }}" + } + }`, + expectedResult: Config{ + CertFile: "/etc/certs/env_cert.crt", + Credentials: "env_credentials", + NestedOption: struct { + URL string `json:"URL"` + }{ + URL: "https://env.testing.com", + }, + }, + expectError: false, + }, + { + name: "Missing environment variable", + envVars: map[string]string{ + "CERT_FILE": "/etc/certs/env_cert.crt", + "CREDENTIALS": "env_credentials", + // "URL" is intentionally missing + }, + jsonTemplate: `{ + "CertFile": "{{ .CERT_FILE }}", + "Credentials": "{{ .CREDENTIALS }}", + "NestedOption": { + "URL": "{{ .URL }}" + } + }`, + expectedResult: Config{ + CertFile: "/etc/certs/env_cert.crt", + Credentials: "env_credentials", + NestedOption: struct { + URL string `json:"URL"` + }{ + URL: "", + }, + }, + expectError: false, + }, + { + name: "Invalid JSON template", + envVars: map[string]string{ + "CERT_FILE": "/etc/certs/env_cert.crt", + "CREDENTIALS": "env_credentials", + "URL": "https://env.testing.com", + }, + jsonTemplate: `{ + "CertFile": "{{ .CERT_FILE }}", + "Credentials": "{{ .CREDENTIALS }", + "NestedOption": { + "URL": "{{ .URL }}" + } + }`, // Note the missing closing brace in "{{ .CREDENTIALS }" + expectedResult: Config{}, + expectError: true, + errorContains: "unexpected \"}\" in operand", + }, + { + name: "No substitutions", + envVars: map[string]string{ + "CERT_FILE": "/etc/certs/env_cert.crt", + "CREDENTIALS": "env_credentials", + "URL": "https://env.testing.com", + }, + jsonTemplate: `{ + "CertFile": "/etc/certs/cert.crt", + "Credentials": "admnlknflkdasdf", + "NestedOption" : { + "URL": "https://testing.com" + } + }`, + expectedResult: Config{ + CertFile: "/etc/certs/cert.crt", + Credentials: "admnlknflkdasdf", + NestedOption: struct { + URL string `json:"URL"` + }{ + URL: "https://testing.com", + }, + }, + expectError: false, + }, + { + name: "Should fail when Invalid characters in variables", + envVars: map[string]string{ + "CERT_FILE": `"/etc/certs/"cert".crt"`, + "CREDENTIALS": `env_credentia{ls}`, + "URL": `https://env.testing.com?param={{value}}`, + }, + jsonTemplate: `{ + "CertFile": "{{ .CERT_FILE }}", + "Credentials": "{{ .CREDENTIALS }}", + "NestedOption": { + "URL": "{{ .URL }}" + } + }`, + expectedResult: Config{ + CertFile: `"/etc/certs/"cert".crt"`, + Credentials: `env_credentia{ls}`, + NestedOption: struct { + URL string `json:"URL"` + }{ + URL: `https://env.testing.com?param={{value}}`, + }, + }, + expectError: true, + }, + } - Describe("Config", func() { - Context("in JSON format", func() { - It("should be written and read successfully", func() { + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + for key, value := range tc.envVars { + t.Setenv(key, value) + } - m := make(map[string]string) - m["key1"] = "value1" - m["key2"] = "value2" + tempFile, err := os.CreateTemp("", "config*.json") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) + } - arr := []string{"value1", "value2"} - - written := &TestConfig{ - SomeMap: m, - SomeArray: arr, - SomeField: 99, + defer func() { + err = os.Remove(tempFile.Name()) + if err != nil { + t.Logf("Failed to remove temp file: %v", err) } + }() - err := util.WriteJson(tmpDir+"/testconfig.json", written) - Expect(err).NotTo(HaveOccurred()) + _, err = tempFile.WriteString(tc.jsonTemplate) + if err != nil { + t.Fatalf("Failed to write to temp file: %v", err) + } + err = tempFile.Close() + if err != nil { + t.Fatalf("Failed to close temp file: %v", err) + } - read, err := util.ReadJson(tmpDir+"/testconfig.json", &TestConfig{}) - Expect(err).NotTo(HaveOccurred()) - Expect(read).NotTo(BeNil()) - Expect(read.(*TestConfig).SomeMap["key1"]).To(BeEquivalentTo(written.SomeMap["key1"])) - Expect(read.(*TestConfig).SomeMap["key2"]).To(BeEquivalentTo(written.SomeMap["key2"])) - Expect(read.(*TestConfig).SomeArray).To(ContainElements(arr)) - Expect(read.(*TestConfig).SomeField).To(BeEquivalentTo(written.SomeField)) + var result Config - }) - }) - }) + _, err = ReadJsonWithEnvSub(tempFile.Name(), &result) - Describe("Copying file contents", func() { - Context("from one file to another", func() { - It("should be successful", func() { - - src := tmpDir + "/copytest_src" - dst := tmpDir + "/copytest_dst" - - err := util.WriteJson(src, []string{"1", "2", "3"}) - Expect(err).NotTo(HaveOccurred()) - - err = util.CopyFileContents(src, dst) - Expect(err).NotTo(HaveOccurred()) - - hashSrc := md5.New() - hashDst := md5.New() - - srcFile, err := os.Open(src) - Expect(err).NotTo(HaveOccurred()) - - dstFile, err := os.Open(dst) - Expect(err).NotTo(HaveOccurred()) - - _, err = io.Copy(hashSrc, srcFile) - Expect(err).NotTo(HaveOccurred()) - - _, err = io.Copy(hashDst, dstFile) - Expect(err).NotTo(HaveOccurred()) - - err = srcFile.Close() - Expect(err).NotTo(HaveOccurred()) - - err = dstFile.Close() - Expect(err).NotTo(HaveOccurred()) - - Expect(hex.EncodeToString(hashSrc.Sum(nil)[:16])).To(BeEquivalentTo(hex.EncodeToString(hashDst.Sum(nil)[:16]))) - }) - }) - }) - - Describe("Handle config file without full path", func() { - Context("config file handling", func() { - It("should be successful", func() { - written := &TestConfig{ - SomeField: 123, + if tc.expectError { + if err == nil { + t.Fatalf("Expected error but got none") } - cfgFile := "test_cfg.json" - defer os.Remove(cfgFile) - - err := util.WriteJson(cfgFile, written) - Expect(err).NotTo(HaveOccurred()) - - read, err := util.ReadJson(cfgFile, &TestConfig{}) - Expect(err).NotTo(HaveOccurred()) - Expect(read).NotTo(BeNil()) - }) + if !strings.Contains(err.Error(), tc.errorContains) { + t.Errorf("Expected error containing '%s', but got '%v'", tc.errorContains, err) + } + } else { + if err != nil { + t.Fatalf("ReadJsonWithEnvSub failed: %v", err) + } + if !reflect.DeepEqual(result, tc.expectedResult) { + t.Errorf("Result does not match expected.\nGot: %+v\nExpected: %+v", result, tc.expectedResult) + } + } }) - }) -}) + } +}