sendnrw/netbird
2
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2026-04-16 15:26:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

[management] Refactor users to use store methods (#2917)

Browse Source 
* Refactor setup key handling to use store methods

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* add lock to get account groups

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* add check for regular user

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* get only required groups for auto-group validation

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* add account lock and return auto groups map on validation

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* refactor account peers update

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor groups to use store methods

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* refactor GetGroupByID and add NewGroupNotFoundError

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* fix tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add AddPeer and RemovePeer methods to Group struct

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Preserve store engine in SqlStore transactions

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Run groups ops in transaction

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* fix missing group removed from setup key activity

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* fix merge

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor posture checks to remove get and save account

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* fix refactor

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* fix tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* fix merge

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* fix sonar

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Change setup key log level to debug for missing group

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Retrieve modified peers once for group events

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor policy get and save account to use store methods

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Retrieve policy groups and posture checks once for validation

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix typo

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add policy tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor anyGroupHasPeers to retrieve all groups once

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor dns settings to use store methods

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add account locking and merge group deletion methods

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor name server groups to use store methods

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add peer store methods

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor ephemeral peers

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add lock for peer store methods

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor peer handlers

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor peer to use store methods

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix typo

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add locks and remove log

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* run peer ops in transaction

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* remove duplicate store method

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* fix peer fields updated after save

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* add tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Use update strength and simplify check

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* prevent changing ruleID when not empty

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* prevent duplicate rules during updates

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* fix tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* fix lint

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor auth middleware

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor account methods and mock

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor user and PAT handling

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Remove db query context and fix get user by id

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix database transaction locking issue

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Use UTC time in test

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add account locks

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix prevent users from creating PATs for other users

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add store locks and prevent fetching setup keys peers when retrieving user peers with empty userID

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add missing tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor test names and remove duplicate TestPostgresql_SavePeerStatus

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add account locks and remove redundant ephemeral check

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Retrieve all groups for peers and restrict groups for regular users

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix merge

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix merge

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* fix merge

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* fix store tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* use account object to get validated peers

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix merge

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Improve peer performance

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Get account direct from store without buffer

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Add get peer groups tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Adjust benchmarks

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Adjust benchmarks

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* [management] Update benchmark workflow (#3181)

* update local benchmark expectations

* update cloud expectations

* Add status error for generic result error

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Use integrated validator direct

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* update expectations

* update expectations

* update expectations

* Refactor peer scheduler to retry every 3 seconds on errors

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* update expectations

* fix validator

* fix validator

* fix validator

* update timeouts

* Refactor ToGroupsInfo to process slices of groups

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* update expectations

* update expectations

* update expectations

* Bump integrations version

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor GetValidatedPeers

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* go mod tidy

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Use peers and groups map for peers validation

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* remove mysql from api benchmark tests

* Fix merge

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix blocked db calls on user auto groups update

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* update expectations

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* update expectations

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Skip user check for system initiated peer deletion

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Remove context in db calls

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* update expectations

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* [management] Improve group peer/resource counting (#3192)

* Fix sonar

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Adjust bench expectations

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Rename GetAccountInfoFromPAT to GetTokenInfo

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fix tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Remove global account lock for ListUsers

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* build userinfo after updating users in db

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* [management] Optimize user bulk deletion  (#3315)

* refactor building user infos

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* fix tests

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* remove unused code

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Refactor GetUsersFromAccount to return a map of UserInfo instead of a slice

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Export BuildUserInfosForAccount to account manager

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Fetch account user info once for bulk users save

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Update user deletion expectations

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Set max open conns for activity store

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

* Update bench expectations

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

---------

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>

---------

Signed-off-by: bcmmbaga <bethuelmbaga12@gmail.com>
Co-authored-by: Pascal Fischer <32096965+pascal-fischer@users.noreply.github.com>
Co-authored-by: Pascal Fischer <pascal@netbird.io>
Co-authored-by: Pedro Costa <550684+pnmcosta@users.noreply.github.com>
This commit is contained in:
Bethuel Mmbaga
2025-02-17 21:43:12 +03:00
committed by GitHub
parent abe8da697c
commit 4cdb2e533a
20 changed files with 1080 additions and 719 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
management/server/http/testing/benchmarks/users_handler_benchmark_test.go
View File

@@ -35,14 +35,14 @@ var benchCasesUsers = map[string]testing_tools.BenchmarkCase{
func BenchmarkUpdateUser(b *testing.B) {
var expectedMetrics = map[string]testing_tools.PerformanceMetrics{
"Users - XS": {MinMsPerOpLocal: 700, MaxMsPerOpLocal: 1000, MinMsPerOpCICD: 1300, MaxMsPerOpCICD: 8000},
"Users - S": {MinMsPerOpLocal: 1, MaxMsPerOpLocal: 5, MinMsPerOpCICD: 4, MaxMsPerOpCICD: 50},
"Users - M": {MinMsPerOpLocal: 20, MaxMsPerOpLocal: 40, MinMsPerOpCICD: 30, MaxMsPerOpCICD: 250},
"Users - L": {MinMsPerOpLocal: 60, MaxMsPerOpLocal: 100, MinMsPerOpCICD: 90, MaxMsPerOpCICD: 700},
"Peers - L": {MinMsPerOpLocal: 300, MaxMsPerOpLocal: 500, MinMsPerOpCICD: 550, MaxMsPerOpCICD: 2400},
"Groups - L": {MinMsPerOpLocal: 400, MaxMsPerOpLocal: 600, MinMsPerOpCICD: 750, MaxMsPerOpCICD: 5000},
"Setup Keys - L": {MinMsPerOpLocal: 50, MaxMsPerOpLocal: 200, MinMsPerOpCICD: 130, MaxMsPerOpCICD: 1000},
"Users - XL": {MinMsPerOpLocal: 350, MaxMsPerOpLocal: 550, MinMsPerOpCICD: 650, MaxMsPerOpCICD: 3500},
"Users - XS": {MinMsPerOpLocal: 100, MaxMsPerOpLocal: 160, MinMsPerOpCICD: 100, MaxMsPerOpCICD: 310},
"Users - S": {MinMsPerOpLocal: 0.3, MaxMsPerOpLocal: 3, MinMsPerOpCICD: 1, MaxMsPerOpCICD: 15},
"Users - M": {MinMsPerOpLocal: 1, MaxMsPerOpLocal: 10, MinMsPerOpCICD: 3, MaxMsPerOpCICD: 20},
"Users - L": {MinMsPerOpLocal: 5, MaxMsPerOpLocal: 20, MinMsPerOpCICD: 2, MaxMsPerOpCICD: 50},
"Peers - L": {MinMsPerOpLocal: 80, MaxMsPerOpLocal: 150, MinMsPerOpCICD: 80, MaxMsPerOpCICD: 310},
"Groups - L": {MinMsPerOpLocal: 10, MaxMsPerOpLocal: 50, MinMsPerOpCICD: 20, MaxMsPerOpCICD: 120},
"Setup Keys - L": {MinMsPerOpLocal: 5, MaxMsPerOpLocal: 20, MinMsPerOpCICD: 2, MaxMsPerOpCICD: 50},
"Users - XL": {MinMsPerOpLocal: 30, MaxMsPerOpLocal: 100, MinMsPerOpCICD: 60, MaxMsPerOpCICD: 280},
}
log.SetOutput(io.Discard)
@@ -118,14 +118,14 @@ func BenchmarkGetOneUser(b *testing.B) {
func BenchmarkGetAllUsers(b *testing.B) {
var expectedMetrics = map[string]testing_tools.PerformanceMetrics{
"Users - XS": {MinMsPerOpLocal: 50, MaxMsPerOpLocal: 90, MinMsPerOpCICD: 60, MaxMsPerOpCICD: 180},
"Users - S": {MinMsPerOpLocal: 0, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 0, MaxMsPerOpCICD: 30},
"Users - M": {MinMsPerOpLocal: 5, MaxMsPerOpLocal: 12, MinMsPerOpCICD: 0, MaxMsPerOpCICD: 30},
"Users - L": {MinMsPerOpLocal: 0, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 0, MaxMsPerOpCICD: 30},
"Peers - L": {MinMsPerOpLocal: 0, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 0, MaxMsPerOpCICD: 30},
"Groups - L": {MinMsPerOpLocal: 0, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 0, MaxMsPerOpCICD: 30},
"Setup Keys - L": {MinMsPerOpLocal: 40, MaxMsPerOpLocal: 140, MinMsPerOpCICD: 60, MaxMsPerOpCICD: 200},
"Users - XL": {MinMsPerOpLocal: 15, MaxMsPerOpLocal: 40, MinMsPerOpCICD: 20, MaxMsPerOpCICD: 90},
"Users - XS": {MinMsPerOpLocal: 0, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 0, MaxMsPerOpCICD: 10},
"Users - S": {MinMsPerOpLocal: 0, MaxMsPerOpLocal: 2, MinMsPerOpCICD: 0, MaxMsPerOpCICD: 10},
"Users - M": {MinMsPerOpLocal: 3, MaxMsPerOpLocal: 10, MinMsPerOpCICD: 5, MaxMsPerOpCICD: 15},
"Users - L": {MinMsPerOpLocal: 10, MaxMsPerOpLocal: 20, MinMsPerOpCICD: 20, MaxMsPerOpCICD: 50},
"Peers - L": {MinMsPerOpLocal: 15, MaxMsPerOpLocal: 25, MinMsPerOpCICD: 20, MaxMsPerOpCICD: 55},
"Groups - L": {MinMsPerOpLocal: 15, MaxMsPerOpLocal: 25, MinMsPerOpCICD: 25, MaxMsPerOpCICD: 55},
"Setup Keys - L": {MinMsPerOpLocal: 15, MaxMsPerOpLocal: 25, MinMsPerOpCICD: 25, MaxMsPerOpCICD: 55},
"Users - XL": {MinMsPerOpLocal: 80, MaxMsPerOpLocal: 120, MinMsPerOpCICD: 100, MaxMsPerOpCICD: 300},
}
log.SetOutput(io.Discard)
@@ -141,7 +141,7 @@ func BenchmarkGetAllUsers(b *testing.B) {
b.ResetTimer()
start := time.Now()
for i := 0; i < b.N; i++ {
req := testing_tools.BuildRequest(b, nil, http.MethodGet, "/api/setup-keys", testing_tools.TestAdminId)
req := testing_tools.BuildRequest(b, nil, http.MethodGet, "/api/users", testing_tools.TestAdminId)
apiHandler.ServeHTTP(recorder, req)
}
@@ -152,14 +152,14 @@ func BenchmarkGetAllUsers(b *testing.B) {
func BenchmarkDeleteUsers(b *testing.B) {
var expectedMetrics = map[string]testing_tools.PerformanceMetrics{
"Users - XS": {MinMsPerOpLocal: 1000, MaxMsPerOpLocal: 1600, MinMsPerOpCICD: 1900, MaxMsPerOpCICD: 11000},
"Users - S": {MinMsPerOpLocal: 15, MaxMsPerOpLocal: 40, MinMsPerOpCICD: 30, MaxMsPerOpCICD: 200},
"Users - M": {MinMsPerOpLocal: 15, MaxMsPerOpLocal: 70, MinMsPerOpCICD: 15, MaxMsPerOpCICD: 230},
"Users - L": {MinMsPerOpLocal: 15, MaxMsPerOpLocal: 45, MinMsPerOpCICD: 30, MaxMsPerOpCICD: 190},
"Peers - L": {MinMsPerOpLocal: 400, MaxMsPerOpLocal: 600, MinMsPerOpCICD: 650, MaxMsPerOpCICD: 1800},
"Groups - L": {MinMsPerOpLocal: 600, MaxMsPerOpLocal: 800, MinMsPerOpCICD: 1200, MaxMsPerOpCICD: 7500},
"Setup Keys - L": {MinMsPerOpLocal: 20, MaxMsPerOpLocal: 200, MinMsPerOpCICD: 40, MaxMsPerOpCICD: 600},
"Users - XL": {MinMsPerOpLocal: 50, MaxMsPerOpLocal: 150, MinMsPerOpCICD: 80, MaxMsPerOpCICD: 400},
"Users - XS": {MinMsPerOpLocal: 0, MaxMsPerOpLocal: 5, MinMsPerOpCICD: 2, MaxMsPerOpCICD: 15},
"Users - S": {MinMsPerOpLocal: 0, MaxMsPerOpLocal: 5, MinMsPerOpCICD: 2, MaxMsPerOpCICD: 15},
"Users - M": {MinMsPerOpLocal: 0, MaxMsPerOpLocal: 5, MinMsPerOpCICD: 2, MaxMsPerOpCICD: 15},
"Users - L": {MinMsPerOpLocal: 0, MaxMsPerOpLocal: 5, MinMsPerOpCICD: 2, MaxMsPerOpCICD: 15},
"Peers - L": {MinMsPerOpLocal: 0, MaxMsPerOpLocal: 5, MinMsPerOpCICD: 2, MaxMsPerOpCICD: 15},
"Groups - L": {MinMsPerOpLocal: 0, MaxMsPerOpLocal: 5, MinMsPerOpCICD: 2, MaxMsPerOpCICD: 15},
"Setup Keys - L": {MinMsPerOpLocal: 0, MaxMsPerOpLocal: 5, MinMsPerOpCICD: 2, MaxMsPerOpCICD: 15},
"Users - XL": {MinMsPerOpLocal: 0, MaxMsPerOpLocal: 5, MinMsPerOpCICD: 2, MaxMsPerOpCICD: 15},
}
log.SetOutput(io.Discard)