Update WireGuard trademark note

This PR brings open-telemetry metrics to the
Management service.
The Management service exposes new HTTP endpoint
/metrics on 8081 port by default.
The port can be changed by specifying
--metrics-port PORT flag when starting the service.

README.md

@@ -1,6 +1,6 @@
 <p align="center">
- <strong>:hatching_chick: New release! NetBird Easy SSH</strong>.
+ <strong>:hatching_chick: New Release! User Invites.</strong>
-  <a href="https://github.com/netbirdio/netbird/releases/tag/v0.8.0">
+  <a href="https://github.com/netbirdio/netbird/releases">
        Learn more
      </a>   
 </p>
@@ -62,9 +62,8 @@ NetBird creates an overlay peer-to-peer network connecting machines automaticall
 -  \[ ] Network Activity Monitoring.
 
 ### Secure peer-to-peer VPN with SSO and MFA in minutes
-<p float="left" align="middle">
+
-  <img src="docs/media/netbird-sso-mfa-demo.gif" width="800"/> 
+https://user-images.githubusercontent.com/700848/197345890-2e2cded5-7b7a-436f-a444-94e80dd24f46.mov
-</p>
 
 **Note**: The `main` branch may be in an *unstable or even broken state* during development. 
 For stable versions, see [releases](https://github.com/netbirdio/netbird/releases).
@@ -104,5 +103,5 @@ See a complete [architecture overview](https://netbird.io/docs/overview/architec
 We use open-source technologies like [WireGuard®](https://www.wireguard.com/), [Pion ICE (WebRTC)](https://github.com/pion/ice), and [Coturn](https://github.com/coturn/coturn). We very much appreciate the work these guys are doing and we'd greatly appreciate if you could support them in any way (e.g. giving a star or a contribution).
 
 ### Legal
- [WireGuard](https://wireguard.com/) is a registered trademark of Jason A. Donenfeld.
+ _WireGuard_ and the _WireGuard_ logo are [registered trademarks](https://www.wireguard.com/trademark-policy/) of Jason A. Donenfeld.
 

client/cmd/testutil.go

@@ -68,12 +68,12 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste
 	}
 
 	peersUpdateManager := mgmt.NewPeersUpdateManager()
-	accountManager, err := mgmt.BuildManager(store, peersUpdateManager, nil)
+	accountManager, err := mgmt.BuildManager(store, peersUpdateManager, nil, "")
 	if err != nil {
 		t.Fatal(err)
 	}
 	turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
-	mgmtServer, err := mgmt.NewServer(config, accountManager, peersUpdateManager, turnManager)
+	mgmtServer, err := mgmt.NewServer(config, accountManager, peersUpdateManager, turnManager, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

client/internal/engine_test.go

@@ -761,12 +761,12 @@ func startManagement(port int, dataDir string) (*grpc.Server, error) {
 		log.Fatalf("failed creating a store: %s: %v", config.Datadir, err)
 	}
 	peersUpdateManager := server.NewPeersUpdateManager()
-	accountManager, err := server.BuildManager(store, peersUpdateManager, nil)
+	accountManager, err := server.BuildManager(store, peersUpdateManager, nil, "")
 	if err != nil {
 		return nil, err
 	}
 	turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
-	mgmtServer, err := server.NewServer(config, accountManager, peersUpdateManager, turnManager)
+	mgmtServer, err := server.NewServer(config, accountManager, peersUpdateManager, turnManager, nil)
 	if err != nil {
 		return nil, err
 	}

go.mod

@@ -18,12 +18,12 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/vishvananda/netlink v1.1.0
 	golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9
-	golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664
+	golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8
 	golang.zx2c4.com/wireguard v0.0.0-20211209221555-9c9e7e272434
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20211215182854-7a385b3431de
 	golang.zx2c4.com/wireguard/windows v0.5.1
 	google.golang.org/grpc v1.43.0
-	google.golang.org/protobuf v1.28.0
+	google.golang.org/protobuf v1.28.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
@@ -39,9 +39,13 @@ require (
 	github.com/libp2p/go-netroute v0.2.0
 	github.com/magiconair/properties v1.8.5
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/prometheus/client_golang v1.13.0
 	github.com/rs/xid v1.3.0
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/stretchr/testify v1.8.0
+	go.opentelemetry.io/otel/exporters/prometheus v0.33.0
+	go.opentelemetry.io/otel/metric v0.33.0
+	go.opentelemetry.io/otel/sdk/metric v0.33.0
 	golang.org/x/net v0.0.0-20220630215102-69896b714898
 	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
 )
@@ -65,11 +69,13 @@ require (
 	github.com/getlantern/ops v0.0.0-20190325191751-d70cb0d6f85f // indirect
 	github.com/go-gl/gl v0.0.0-20210813123233-e4099ee2221f // indirect
 	github.com/go-gl/glfw/v3.3/glfw v0.0.0-20211024062804-40e447a793be // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-redis/redis/v8 v8.11.5 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/godbus/dbus/v5 v5.0.4 // indirect
 	github.com/goki/freetype v0.0.0-20181231101311-fa8a33aabaff // indirect
-	github.com/google/go-cmp v0.5.7 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/josharian/native v0.0.0-20200817173448-b6b71def0850 // indirect
@@ -89,20 +95,22 @@ require (
 	github.com/pion/turn/v2 v2.0.8 // indirect
 	github.com/pion/udp v0.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.12.2 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.33.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/rogpeppe/go-internal v1.8.0 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/srwiley/oksvg v0.0.0-20200311192757-870daf9aa564 // indirect
 	github.com/srwiley/rasterx v0.0.0-20200120212402-85cb7272f5e9 // indirect
 	github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect
 	github.com/yuin/goldmark v1.4.1 // indirect
+	go.opentelemetry.io/otel v1.11.1 // indirect
+	go.opentelemetry.io/otel/sdk v1.11.1 // indirect
+	go.opentelemetry.io/otel/trace v1.11.1 // indirect
 	golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf // indirect
 	golang.org/x/image v0.0.0-20200430140353-33d19683fad8 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
 	golang.org/x/text v0.3.8-0.20211105212822-18b340fc7af2 // indirect
 	golang.org/x/tools v0.1.10 // indirect
 	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect

go.sum

@@ -202,6 +202,11 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
+github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -278,8 +283,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -542,8 +547,8 @@ github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
+github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU=
-github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -554,15 +559,16 @@ github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8b
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.33.0 h1:rHgav/0a6+uYgGdNt3jwz8FNSesO/Hsang3O0T9A5SE=
+github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
-github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE=
+github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
+github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
@@ -648,6 +654,18 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/otel v1.11.1 h1:4WLLAmcfkmDk2ukNXJyq3/kiz/3UzCaYq6PskJsaou4=
+go.opentelemetry.io/otel v1.11.1/go.mod h1:1nNhXBbWSD0nsL38H6btgnFN2k4i0sNLHNNMZMSbUGE=
+go.opentelemetry.io/otel/exporters/prometheus v0.33.0 h1:xXhPj7SLKWU5/Zd4Hxmd+X1C4jdmvc0Xy+kvjFx2z60=
+go.opentelemetry.io/otel/exporters/prometheus v0.33.0/go.mod h1:ZSmYfKdYWEdSDBB4njLBIwTf4AU2JNsH3n2quVQDebI=
+go.opentelemetry.io/otel/metric v0.33.0 h1:xQAyl7uGEYvrLAiV/09iTJlp1pZnQ9Wl793qbVvED1E=
+go.opentelemetry.io/otel/metric v0.33.0/go.mod h1:QlTYc+EnYNq/M2mNk1qDDMRLpqCOj2f/r5c7Fd5FYaI=
+go.opentelemetry.io/otel/sdk v1.11.1 h1:F7KmQgoHljhUuJyA+9BiU+EkJfyX5nVVF4wyzWZpKxs=
+go.opentelemetry.io/otel/sdk v1.11.1/go.mod h1:/l3FE4SupHJ12TduVjUkZtlfFqDCQJlOlithYrdktys=
+go.opentelemetry.io/otel/sdk/metric v0.33.0 h1:oTqyWfksgKoJmbrs2q7O7ahkJzt+Ipekihf8vhpa9qo=
+go.opentelemetry.io/otel/sdk/metric v0.33.0/go.mod h1:xdypMeA21JBOvjjzDUtD0kzIcHO/SPez+a8HOzJPGp0=
+go.opentelemetry.io/otel/trace v1.11.1 h1:ofxdnzsNrGBYXbP7t7zpUK281+go5rF7dvdIZXF8gdQ=
+go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
@@ -809,8 +827,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -915,8 +934,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664 h1:wEZYwx+kK+KlZ0hpvP2Ls1Xr4+RWnlzGFwPP0aiDjIU=
+golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 h1:h+EGohizhe9XlX18rfpa8k8RAc5XyaeamM+0VHRd4lc=
-golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM=
@@ -1162,8 +1181,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
-google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

infrastructure_files/base.setup.env

@@ -10,6 +10,8 @@ NETBIRD_MGMT_API_ENDPOINT=https://$NETBIRD_DOMAIN:$NETBIRD_MGMT_API_PORT
 NETBIRD_MGMT_API_CERT_FILE="/etc/letsencrypt/live/$NETBIRD_DOMAIN/fullchain.pem"
 # Management Certficate key file path.
 NETBIRD_MGMT_API_CERT_KEY_FILE="/etc/letsencrypt/live/$NETBIRD_DOMAIN/privkey.pem"
+# By default Management single account mode is enabled and domain set to $NETBIRD_DOMAIN, you may want to set this to your user's email domain
+NETBIRD_MGMT_SINGLE_ACCOUNT_MODE_DOMAIN=$NETBIRD_DOMAIN
 
 # Turn credentials
 

infrastructure_files/docker-compose.yml.tmpl

@@ -48,7 +48,7 @@ services:
   #     # port and command for Let's Encrypt validation without dashboard container
   #   - 443:443
   #    command: ["--letsencrypt-domain", "$NETBIRD_DOMAIN", "--log-file", "console"]
-    command: ["--port", "443", "--log-file", "console", "--disable-anonymous-metrics=$NETBIRD_DISABLE_ANONYMOUS_METRICS"]
+    command: ["--port", "443", "--log-file", "console", "--disable-anonymous-metrics=$NETBIRD_DISABLE_ANONYMOUS_METRICS", "--single-account-mode-domain=$NETBIRD_MGMT_SINGLE_ACCOUNT_MODE_DOMAIN"]
   # Coturn
   coturn:
     image: coturn/coturn

management/client/client_test.go

@@ -55,12 +55,12 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) {
 	}
 
 	peersUpdateManager := mgmt.NewPeersUpdateManager()
-	accountManager, err := mgmt.BuildManager(store, peersUpdateManager, nil)
+	accountManager, err := mgmt.BuildManager(store, peersUpdateManager, nil, "")
 	if err != nil {
 		t.Fatal(err)
 	}
 	turnManager := mgmt.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
-	mgmtServer, err := mgmt.NewServer(config, accountManager, peersUpdateManager, turnManager)
+	mgmtServer, err := mgmt.NewServer(config, accountManager, peersUpdateManager, turnManager, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

management/cmd/management.go

@@ -10,6 +10,7 @@ import (
 	"github.com/google/uuid"
 	httpapi "github.com/netbirdio/netbird/management/server/http"
 	"github.com/netbirdio/netbird/management/server/metrics"
+	"github.com/netbirdio/netbird/management/server/telemetry"
 	"golang.org/x/crypto/acme/autocert"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
@@ -41,11 +42,13 @@ import (
 const ManagementLegacyPort = 33073
 
 var (
-	mgmtPort              int
+	mgmtPort                int
-	mgmtLetsencryptDomain string
+	mgmtMetricsPort         int
-	certFile              string
+	mgmtLetsencryptDomain   string
-	certKey               string
+	mgmtSingleAccModeDomain string
-	config                *server.Config
+	certFile                string
+	certKey                 string
+	config                  *server.Config
 
 	kaep = keepalive.EnforcementPolicy{
 		MinTime:             15 * time.Second,
@@ -113,15 +116,27 @@ var (
 			}
 			peersUpdateManager := server.NewPeersUpdateManager()
 
+			appMetrics, err := telemetry.NewDefaultAppMetrics(cmd.Context())
+			if err != nil {
+				return err
+			}
+			err = appMetrics.Expose(mgmtMetricsPort, "/metrics")
+			if err != nil {
+				return err
+			}
+
 			var idpManager idp.Manager
 			if config.IdpManagerConfig != nil {
-				idpManager, err = idp.NewManager(*config.IdpManagerConfig)
+				idpManager, err = idp.NewManager(*config.IdpManagerConfig, appMetrics)
 				if err != nil {
 					return fmt.Errorf("failed retrieving a new idp manager with err: %v", err)
 				}
 			}
 
-			accountManager, err := server.BuildManager(store, peersUpdateManager, idpManager)
+			if disableSingleAccMode {
+				mgmtSingleAccModeDomain = ""
+			}
+			accountManager, err := server.BuildManager(store, peersUpdateManager, idpManager, mgmtSingleAccModeDomain)
 			if err != nil {
 				return fmt.Errorf("failed to build default manager: %v", err)
 			}
@@ -151,14 +166,14 @@ var (
 				tlsEnabled = true
 			}
 
-			httpAPIHandler, err := httpapi.APIHandler(accountManager,
+			httpAPIHandler, err := httpapi.APIHandler(accountManager, config.HttpConfig.AuthIssuer,
-				config.HttpConfig.AuthIssuer, config.HttpConfig.AuthAudience, config.HttpConfig.AuthKeysLocation)
+				config.HttpConfig.AuthAudience, config.HttpConfig.AuthKeysLocation, appMetrics)
 			if err != nil {
 				return fmt.Errorf("failed creating HTTP API handler: %v", err)
 			}
 
 			gRPCAPIHandler := grpc.NewServer(gRPCOpts...)
-			srv, err := server.NewServer(config, accountManager, peersUpdateManager, turnManager)
+			srv, err := server.NewServer(config, accountManager, peersUpdateManager, turnManager, appMetrics)
 			if err != nil {
 				return fmt.Errorf("failed creating gRPC API handler: %v", err)
 			}
@@ -225,6 +240,7 @@ var (
 			SetupCloseHandler()
 
 			<-stopCh
+			_ = appMetrics.Close()
 			_ = listener.Close()
 			if certManager != nil {
 				_ = certManager.Listener().Close()
@@ -427,7 +443,7 @@ func loadTLSConfig(certFile string, certKey string) (*tls.Config, error) {
 		return nil, err
 	}
 
-	// Create the credentials and return it
+	// NewDefaultAppMetrics the credentials and return it
 	config := &tls.Config{
 		Certificates: []tls.Certificate{serverCert},
 		ClientAuth:   tls.NoClientCert,

management/cmd/root.go

@@ -13,21 +13,23 @@ const (
 )
 
 var (
-	defaultMgmtConfigDir    string
+	defaultMgmtConfigDir       string
-	defaultMgmtDataDir      string
+	defaultMgmtDataDir         string
-	defaultMgmtConfig       string
+	defaultMgmtConfig          string
-	defaultLogDir           string
+	defaultSingleAccModeDomain string
-	defaultLogFile          string
+	defaultLogDir              string
-	oldDefaultMgmtConfigDir string
+	defaultLogFile             string
-	oldDefaultMgmtDataDir   string
+	oldDefaultMgmtConfigDir    string
-	oldDefaultMgmtConfig    string
+	oldDefaultMgmtDataDir      string
-	oldDefaultLogDir        string
+	oldDefaultMgmtConfig       string
-	oldDefaultLogFile       string
+	oldDefaultLogDir           string
-	mgmtDataDir             string
+	oldDefaultLogFile          string
-	mgmtConfig              string
+	mgmtDataDir                string
-	logLevel                string
+	mgmtConfig                 string
-	logFile                 string
+	logLevel                   string
-	disableMetrics          bool
+	logFile                    string
+	disableMetrics             bool
+	disableSingleAccMode       bool
 
 	rootCmd = &cobra.Command{
 		Use:   "netbird-mgmt",
@@ -48,6 +50,7 @@ func init() {
 	stopCh = make(chan int)
 
 	defaultMgmtDataDir = "/var/lib/netbird/"
+	defaultSingleAccModeDomain = "netbird.selfhosted"
 	defaultMgmtConfigDir = "/etc/netbird"
 	defaultLogDir = "/var/log/netbird"
 
@@ -62,9 +65,12 @@ func init() {
 	oldDefaultLogFile = oldDefaultLogDir + "/management.log"
 
 	mgmtCmd.Flags().IntVar(&mgmtPort, "port", 80, "server port to listen on (defaults to 443 if TLS is enabled, 80 otherwise")
+	mgmtCmd.Flags().IntVar(&mgmtMetricsPort, "metrics-port", 8081, "metrics endpoint http port. Metrics are accessible under host:metrics-port/metrics")
 	mgmtCmd.Flags().StringVar(&mgmtDataDir, "datadir", defaultMgmtDataDir, "server data directory location")
 	mgmtCmd.Flags().StringVar(&mgmtConfig, "config", defaultMgmtConfig, "Netbird config file location. Config params specified via command line (e.g. datadir) have a precedence over configuration from this file")
 	mgmtCmd.Flags().StringVar(&mgmtLetsencryptDomain, "letsencrypt-domain", "", "a domain to issue Let's Encrypt certificate for. Enables TLS using Let's Encrypt. Will fetch and renew certificate, and run the server with TLS")
+	mgmtCmd.Flags().StringVar(&mgmtSingleAccModeDomain, "single-account-mode-domain", defaultSingleAccModeDomain, "Enables single account mode. This means that all the users will be under the same account grouped by the specified domain. If the installation has more than one account, the property is ineffective. Enabled by default with the default domain "+defaultSingleAccModeDomain)
+	mgmtCmd.Flags().BoolVar(&disableSingleAccMode, "disable-single-account-mode", false, "If set to true, disables single account mode. The --single-account-mode-domain property will be ignored and every new user will have a separate NetBird account.")
 	mgmtCmd.Flags().StringVar(&certFile, "cert-file", "", "Location of your SSL certificate. Can be used when you have an existing certificate and don't want a new certificate be generated automatically. If letsencrypt-domain is specified this property has no effect")
 	mgmtCmd.Flags().StringVar(&certKey, "cert-key", "", "Location of your SSL certificate private key. Can be used when you have an existing certificate and don't want a new certificate be generated automatically. If letsencrypt-domain is specified this property has no effect")
 	mgmtCmd.Flags().BoolVar(&disableMetrics, "disable-anonymous-metrics", false, "disables push of anonymous usage metrics to NetBird")

management/server/account.go

@@ -106,6 +106,13 @@ type DefaultAccountManager struct {
 	idpManager         idp.Manager
 	cacheManager       cache.CacheInterface[[]*idp.UserData]
 	ctx                context.Context
+
+	// singleAccountMode indicates whether the instance has a single account.
+	// If true, then every new user will end up under the same account.
+	// This value will be set to false if management service has more than one account.
+	singleAccountMode bool
+	// singleAccountModeDomain is a domain to use in singleAccountMode setup
+	singleAccountModeDomain string
 }
 
 // Account represents a unique account of the system
@@ -195,9 +202,8 @@ func (a *Account) GetGroupAll() (*Group, error) {
 }
 
 // BuildManager creates a new DefaultAccountManager with a provided Store
-func BuildManager(
+func BuildManager(store Store, peersUpdateManager *PeersUpdateManager, idpManager idp.Manager,
-	store Store, peersUpdateManager *PeersUpdateManager, idpManager idp.Manager,
+	singleAccountModeDomain string) (*DefaultAccountManager, error) {
-) (*DefaultAccountManager, error) {
 	am := &DefaultAccountManager{
 		Store:              store,
 		mux:                sync.Mutex{},
@@ -207,11 +213,20 @@ func BuildManager(
 		cacheMux:           sync.Mutex{},
 		cacheLoading:       map[string]chan struct{}{},
 	}
+	allAccounts := store.GetAllAccounts()
+	// enable single account mode only if configured by user and number of existing accounts is not grater than 1
+	am.singleAccountMode = singleAccountModeDomain != "" && len(allAccounts) <= 1
+	if am.singleAccountMode {
+		am.singleAccountModeDomain = singleAccountModeDomain
+		log.Infof("single account mode enabled, accounts number %d", len(allAccounts))
+	} else {
+		log.Infof("single account mode disabled, accounts number %d", len(allAccounts))
+	}
 
-	// if account has not default group
+	// if account doesn't have a default group
 	// we create 'all' group and add all peers into it
 	// also we create default rule with source as destination
-	for _, account := range store.GetAllAccounts() {
+	for _, account := range allAccounts {
 		_, err := account.GetGroupAll()
 		if err != nil {
 			addAllGroup(account)
@@ -221,10 +236,10 @@ func BuildManager(
 		}
 	}
 
-	gocacheClient := gocache.New(CacheExpirationMax, 30*time.Minute)
+	goCacheClient := gocache.New(CacheExpirationMax, 30*time.Minute)
-	gocacheStore := cacheStore.NewGoCache(gocacheClient)
+	goCacheStore := cacheStore.NewGoCache(goCacheClient)
 
-	am.cacheManager = cache.NewLoadable[[]*idp.UserData](am.loadAccount, cache.New[[]*idp.UserData](gocacheStore))
+	am.cacheManager = cache.NewLoadable[[]*idp.UserData](am.loadAccount, cache.New[[]*idp.UserData](goCacheStore))
 
 	if !isNil(am.idpManager) {
 		go func() {
@@ -585,7 +600,7 @@ func (am *DefaultAccountManager) redeemInvite(account *Account, userID string) e
 		return status.Errorf(codes.NotFound, "user %s not found in the IdP", userID)
 	}
 
-	if user.AppMetadata.WTPendingInvite {
+	if user.AppMetadata.WTPendingInvite != nil && *user.AppMetadata.WTPendingInvite {
 		log.Infof("redeeming invite for user %s account %s", userID, account.Id)
 		// User has already logged in, meaning that IdP should have set wt_pending_invite to false.
 		// Our job is to just reload cache.
@@ -604,6 +619,15 @@ func (am *DefaultAccountManager) redeemInvite(account *Account, userID string) e
 
 // GetAccountFromToken returns an account associated with this token
 func (am *DefaultAccountManager) GetAccountFromToken(claims jwtclaims.AuthorizationClaims) (*Account, error) {
+
+	if am.singleAccountMode && am.singleAccountModeDomain != "" {
+		// This section is mostly related to self-hosted installations.
+		// We override incoming domain claims to group users under a single account.
+		claims.Domain = am.singleAccountModeDomain
+		claims.DomainCategory = PrivateCategory
+		log.Infof("overriding JWT Domain and DomainCategory claims since single account mode is enabled")
+	}
+
 	account, err := am.getAccountWithAuthorizationClaims(claims)
 	if err != nil {
 		return nil, err

management/server/account_test.go

@@ -962,7 +962,7 @@ func createManager(t *testing.T) (*DefaultAccountManager, error) {
 	if err != nil {
 		return nil, err
 	}
-	return BuildManager(store, NewPeersUpdateManager(), nil)
+	return BuildManager(store, NewPeersUpdateManager(), nil, "")
 }
 
 func createStore(t *testing.T) (Store, error) {

management/server/grpcserver.go

@@ -3,6 +3,7 @@ package server
 import (
 	"context"
 	"fmt"
+	"github.com/netbirdio/netbird/management/server/telemetry"
 	"github.com/netbirdio/netbird/route"
 	gPeer "google.golang.org/grpc/peer"
 	"strings"
@@ -30,10 +31,12 @@ type GRPCServer struct {
 	config                 *Config
 	turnCredentialsManager TURNCredentialsManager
 	jwtMiddleware          *middleware.JWTMiddleware
+	appMetrics             telemetry.AppMetrics
 }
 
 // NewServer creates a new Management server
-func NewServer(config *Config, accountManager AccountManager, peersUpdateManager *PeersUpdateManager, turnCredentialsManager TURNCredentialsManager) (*GRPCServer, error) {
+func NewServer(config *Config, accountManager AccountManager, peersUpdateManager *PeersUpdateManager,
+	turnCredentialsManager TURNCredentialsManager, appMetrics telemetry.AppMetrics) (*GRPCServer, error) {
 	key, err := wgtypes.GeneratePrivateKey()
 	if err != nil {
 		return nil, err
@@ -53,6 +56,16 @@ func NewServer(config *Config, accountManager AccountManager, peersUpdateManager
 		log.Debug("unable to use http config to create new jwt middleware")
 	}
 
+	if appMetrics != nil {
+		// update gauge based on number of connected peers which is equal to open gRPC streams
+		err = appMetrics.GRPCMetrics().RegisterConnectedStreams(func() int64 {
+			return int64(len(peersUpdateManager.peerChannels))
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return &GRPCServer{
 		wgKey: key,
 		// peerKey -> event channel
@@ -61,11 +74,15 @@ func NewServer(config *Config, accountManager AccountManager, peersUpdateManager
 		config:                 config,
 		turnCredentialsManager: turnCredentialsManager,
 		jwtMiddleware:          jwtMiddleware,
+		appMetrics:             appMetrics,
 	}, nil
 }
 
 func (s *GRPCServer) GetServerKey(ctx context.Context, req *proto.Empty) (*proto.ServerKeyResponse, error) {
 	// todo introduce something more meaningful with the key expiration/rotation
+	if s.appMetrics != nil {
+		s.appMetrics.GRPCMetrics().CountGetKeyRequest()
+	}
 	now := time.Now().Add(24 * time.Hour)
 	secs := int64(now.Second())
 	nanos := int32(now.Nanosecond())
@@ -80,6 +97,9 @@ func (s *GRPCServer) GetServerKey(ctx context.Context, req *proto.Empty) (*proto
 // Sync validates the existence of a connecting peer, sends an initial state (all available for the connecting peers) and
 // notifies the connected peer of any updates (e.g. new peers under the same account)
 func (s *GRPCServer) Sync(req *proto.EncryptedMessage, srv proto.ManagementService_SyncServer) error {
+	if s.appMetrics != nil {
+		s.appMetrics.GRPCMetrics().CountSyncRequest()
+	}
 	p, ok := gRPCPeer.FromContext(srv.Context())
 	if ok {
 		log.Debugf("Sync request from peer [%s] [%s]", req.WgPubKey, p.Addr.String())
@@ -259,6 +279,9 @@ func (s *GRPCServer) registerPeer(peerKey wgtypes.Key, req *proto.LoginRequest)
 // In case it isn't, the endpoint checks whether setup key is provided within the request and tries to register a peer.
 // In case of the successful registration login is also successful
 func (s *GRPCServer) Login(ctx context.Context, req *proto.EncryptedMessage) (*proto.EncryptedMessage, error) {
+	if s.appMetrics != nil {
+		s.appMetrics.GRPCMetrics().CountLoginRequest()
+	}
 	p, ok := gRPCPeer.FromContext(ctx)
 	if ok {
 		log.Debugf("Login request from peer [%s] [%s]", req.WgPubKey, p.Addr.String())

management/server/http/handler.go

@@ -4,12 +4,14 @@ import (
 	"github.com/gorilla/mux"
 	s "github.com/netbirdio/netbird/management/server"
 	"github.com/netbirdio/netbird/management/server/http/middleware"
+	"github.com/netbirdio/netbird/management/server/telemetry"
 	"github.com/rs/cors"
 	"net/http"
 )
 
 // APIHandler creates the Management service HTTP API handler registering all the available endpoints.
-func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience string, authKeysLocation string) (http.Handler, error) {
+func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience string, authKeysLocation string,
+	appMetrics telemetry.AppMetrics) (http.Handler, error) {
 	jwtMiddleware, err := middleware.NewJwtMiddleware(
 		authIssuer,
 		authAudience,
@@ -25,8 +27,11 @@ func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience
 		authAudience,
 		accountManager.IsUserAdmin)
 
-	apiHandler := mux.NewRouter()
+	rootRouter := mux.NewRouter()
-	apiHandler.Use(corsMiddleware.Handler, jwtMiddleware.Handler, acMiddleware.Handler)
+	metricsMiddleware := appMetrics.HTTPMiddleware()
+
+	apiHandler := rootRouter.PathPrefix("/api").Subrouter()
+	apiHandler.Use(metricsMiddleware.Handler, corsMiddleware.Handler, jwtMiddleware.Handler, acMiddleware.Handler)
 
 	groupsHandler := NewGroups(accountManager, authAudience)
 	rulesHandler := NewRules(accountManager, authAudience)
@@ -36,46 +41,67 @@ func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience
 	routesHandler := NewRoutes(accountManager, authAudience)
 	nameserversHandler := NewNameservers(accountManager, authAudience)
 
-	apiHandler.HandleFunc("/api/peers", peersHandler.GetPeers).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/peers", peersHandler.GetPeers).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/peers/{id}", peersHandler.HandlePeer).
+	apiHandler.HandleFunc("/peers/{id}", peersHandler.HandlePeer).
 		Methods("GET", "PUT", "DELETE", "OPTIONS")
-	apiHandler.HandleFunc("/api/users", userHandler.GetUsers).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/users", userHandler.GetUsers).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/users/{id}", userHandler.UpdateUser).Methods("PUT", "OPTIONS")
+	apiHandler.HandleFunc("/users/{id}", userHandler.UpdateUser).Methods("PUT", "OPTIONS")
-	apiHandler.HandleFunc("/api/users", userHandler.CreateUserHandler).Methods("POST", "OPTIONS")
+	apiHandler.HandleFunc("/users", userHandler.CreateUserHandler).Methods("POST", "OPTIONS")
 
-	apiHandler.HandleFunc("/api/setup-keys", keysHandler.GetAllSetupKeysHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/setup-keys", keysHandler.GetAllSetupKeysHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/setup-keys", keysHandler.CreateSetupKeyHandler).Methods("POST", "OPTIONS")
+	apiHandler.HandleFunc("/setup-keys", keysHandler.CreateSetupKeyHandler).Methods("POST", "OPTIONS")
-	apiHandler.HandleFunc("/api/setup-keys/{id}", keysHandler.GetSetupKeyHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/setup-keys/{id}", keysHandler.GetSetupKeyHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/setup-keys/{id}", keysHandler.UpdateSetupKeyHandler).Methods("PUT", "OPTIONS")
+	apiHandler.HandleFunc("/setup-keys/{id}", keysHandler.UpdateSetupKeyHandler).Methods("PUT", "OPTIONS")
 
-	apiHandler.HandleFunc("/api/rules", rulesHandler.GetAllRulesHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/rules", rulesHandler.GetAllRulesHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/rules", rulesHandler.CreateRuleHandler).Methods("POST", "OPTIONS")
+	apiHandler.HandleFunc("/rules", rulesHandler.CreateRuleHandler).Methods("POST", "OPTIONS")
-	apiHandler.HandleFunc("/api/rules/{id}", rulesHandler.UpdateRuleHandler).Methods("PUT", "OPTIONS")
+	apiHandler.HandleFunc("/rules/{id}", rulesHandler.UpdateRuleHandler).Methods("PUT", "OPTIONS")
-	apiHandler.HandleFunc("/api/rules/{id}", rulesHandler.PatchRuleHandler).Methods("PATCH", "OPTIONS")
+	apiHandler.HandleFunc("/rules/{id}", rulesHandler.PatchRuleHandler).Methods("PATCH", "OPTIONS")
-	apiHandler.HandleFunc("/api/rules/{id}", rulesHandler.GetRuleHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/rules/{id}", rulesHandler.GetRuleHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/rules/{id}", rulesHandler.DeleteRuleHandler).Methods("DELETE", "OPTIONS")
+	apiHandler.HandleFunc("/rules/{id}", rulesHandler.DeleteRuleHandler).Methods("DELETE", "OPTIONS")
 
-	apiHandler.HandleFunc("/api/groups", groupsHandler.GetAllGroupsHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/groups", groupsHandler.GetAllGroupsHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/groups", groupsHandler.CreateGroupHandler).Methods("POST", "OPTIONS")
+	apiHandler.HandleFunc("/groups", groupsHandler.CreateGroupHandler).Methods("POST", "OPTIONS")
-	apiHandler.HandleFunc("/api/groups/{id}", groupsHandler.UpdateGroupHandler).Methods("PUT", "OPTIONS")
+	apiHandler.HandleFunc("/groups/{id}", groupsHandler.UpdateGroupHandler).Methods("PUT", "OPTIONS")
-	apiHandler.HandleFunc("/api/groups/{id}", groupsHandler.PatchGroupHandler).Methods("PATCH", "OPTIONS")
+	apiHandler.HandleFunc("/groups/{id}", groupsHandler.PatchGroupHandler).Methods("PATCH", "OPTIONS")
-	apiHandler.HandleFunc("/api/groups/{id}", groupsHandler.GetGroupHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/groups/{id}", groupsHandler.GetGroupHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/groups/{id}", groupsHandler.DeleteGroupHandler).Methods("DELETE", "OPTIONS")
+	apiHandler.HandleFunc("/groups/{id}", groupsHandler.DeleteGroupHandler).Methods("DELETE", "OPTIONS")
 
-	apiHandler.HandleFunc("/api/routes", routesHandler.GetAllRoutesHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/routes", routesHandler.GetAllRoutesHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/routes", routesHandler.CreateRouteHandler).Methods("POST", "OPTIONS")
+	apiHandler.HandleFunc("/routes", routesHandler.CreateRouteHandler).Methods("POST", "OPTIONS")
-	apiHandler.HandleFunc("/api/routes/{id}", routesHandler.UpdateRouteHandler).Methods("PUT", "OPTIONS")
+	apiHandler.HandleFunc("/routes/{id}", routesHandler.UpdateRouteHandler).Methods("PUT", "OPTIONS")
-	apiHandler.HandleFunc("/api/routes/{id}", routesHandler.PatchRouteHandler).Methods("PATCH", "OPTIONS")
+	apiHandler.HandleFunc("/routes/{id}", routesHandler.PatchRouteHandler).Methods("PATCH", "OPTIONS")
-	apiHandler.HandleFunc("/api/routes/{id}", routesHandler.GetRouteHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/routes/{id}", routesHandler.GetRouteHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/routes/{id}", routesHandler.DeleteRouteHandler).Methods("DELETE", "OPTIONS")
+	apiHandler.HandleFunc("/routes/{id}", routesHandler.DeleteRouteHandler).Methods("DELETE", "OPTIONS")
 
-	apiHandler.HandleFunc("/api/dns/nameservers", nameserversHandler.GetAllNameserversHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/dns/nameservers", nameserversHandler.GetAllNameserversHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/dns/nameservers", nameserversHandler.CreateNameserverGroupHandler).Methods("POST", "OPTIONS")
+	apiHandler.HandleFunc("/dns/nameservers", nameserversHandler.CreateNameserverGroupHandler).Methods("POST", "OPTIONS")
-	apiHandler.HandleFunc("/api/dns/nameservers/{id}", nameserversHandler.UpdateNameserverGroupHandler).Methods("PUT", "OPTIONS")
+	apiHandler.HandleFunc("/dns/nameservers/{id}", nameserversHandler.UpdateNameserverGroupHandler).Methods("PUT", "OPTIONS")
-	apiHandler.HandleFunc("/api/dns/nameservers/{id}", nameserversHandler.PatchNameserverGroupHandler).Methods("PATCH", "OPTIONS")
+	apiHandler.HandleFunc("/dns/nameservers/{id}", nameserversHandler.PatchNameserverGroupHandler).Methods("PATCH", "OPTIONS")
-	apiHandler.HandleFunc("/api/dns/nameservers/{id}", nameserversHandler.GetNameserverGroupHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/dns/nameservers/{id}", nameserversHandler.GetNameserverGroupHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/dns/nameservers/{id}", nameserversHandler.DeleteNameserverGroupHandler).Methods("DELETE", "OPTIONS")
+	apiHandler.HandleFunc("/dns/nameservers/{id}", nameserversHandler.DeleteNameserverGroupHandler).Methods("DELETE", "OPTIONS")
 
-	return apiHandler, nil
+	err = apiHandler.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error {
+		methods, err := route.GetMethods()
+		if err != nil {
+			return err
+		}
+		for _, method := range methods {
+			template, err := route.GetPathTemplate()
+			if err != nil {
+				return err
+			}
+			err = metricsMiddleware.AddHTTPRequestResponseCounter(template, method)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return rootRouter, nil
 
 }

management/server/http/middleware/jwt.go

@@ -186,7 +186,7 @@ func (m *JWTMiddleware) CheckJWTFromRequest(w http.ResponseWriter, r *http.Reque
 
 	validatedToken, err := m.ValidateAndParse(token)
 	if err != nil {
-		m.Options.ErrorHandler(w, r, "The token isn't valid")
+		m.Options.ErrorHandler(w, r, err.Error())
 		return err
 	}
 

management/server/idp/auth0.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/netbirdio/netbird/management/server/telemetry"
 	"io"
 	"net/http"
 	"net/url"
@@ -24,6 +25,7 @@ type Auth0Manager struct {
 	httpClient  ManagerHTTPClient
 	credentials ManagerCredentials
 	helper      ManagerHelper
+	appMetrics  telemetry.AppMetrics
 }
 
 // Auth0ClientConfig auth0 manager client configurations
@@ -51,6 +53,7 @@ type Auth0Credentials struct {
 	httpClient   ManagerHTTPClient
 	jwtToken     JWTToken
 	mux          sync.Mutex
+	appMetrics   telemetry.AppMetrics
 }
 
 // createUserRequest is a user create request
@@ -106,7 +109,7 @@ type auth0Profile struct {
 }
 
 // NewAuth0Manager creates a new instance of the Auth0Manager
-func NewAuth0Manager(config Auth0ClientConfig) (*Auth0Manager, error) {
+func NewAuth0Manager(config Auth0ClientConfig, appMetrics telemetry.AppMetrics) (*Auth0Manager, error) {
 
 	httpTransport := http.DefaultTransport.(*http.Transport).Clone()
 	httpTransport.MaxIdleConns = 5
@@ -134,12 +137,15 @@ func NewAuth0Manager(config Auth0ClientConfig) (*Auth0Manager, error) {
 		clientConfig: config,
 		httpClient:   httpClient,
 		helper:       helper,
+		appMetrics:   appMetrics,
 	}
+
 	return &Auth0Manager{
 		authIssuer:  config.AuthIssuer,
 		credentials: credentials,
 		httpClient:  httpClient,
 		helper:      helper,
+		appMetrics:  appMetrics,
 	}, nil
 }
 
@@ -170,6 +176,9 @@ func (c *Auth0Credentials) requestJWTToken() (*http.Response, error) {
 
 	res, err = c.httpClient.Do(req)
 	if err != nil {
+		if c.appMetrics != nil {
+			c.appMetrics.IDPMetrics().CountRequestError()
+		}
 		return res, err
 	}
 
@@ -214,6 +223,10 @@ func (c *Auth0Credentials) Authenticate() (JWTToken, error) {
 	c.mux.Lock()
 	defer c.mux.Unlock()
 
+	if c.appMetrics != nil {
+		c.appMetrics.IDPMetrics().CountAuthenticate()
+	}
+
 	// If jwtToken has an expires time and we have enough time to do a request return immediately
 	if c.jwtStillValid() {
 		return c.jwtToken, nil
@@ -287,9 +300,16 @@ func (am *Auth0Manager) GetAccount(accountID string) ([]*UserData, error) {
 
 		res, err := am.httpClient.Do(req)
 		if err != nil {
+			if am.appMetrics != nil {
+				am.appMetrics.IDPMetrics().CountRequestError()
+			}
 			return nil, err
 		}
 
+		if am.appMetrics != nil {
+			am.appMetrics.IDPMetrics().CountGetAccount()
+		}
+
 		body, err := io.ReadAll(res.Body)
 		if err != nil {
 			return nil, err
@@ -342,9 +362,16 @@ func (am *Auth0Manager) GetUserDataByID(userID string, appMetadata AppMetadata)
 
 	res, err := am.httpClient.Do(req)
 	if err != nil {
+		if am.appMetrics != nil {
+			am.appMetrics.IDPMetrics().CountRequestError()
+		}
 		return nil, err
 	}
 
+	if am.appMetrics != nil {
+		am.appMetrics.IDPMetrics().CountGetUserDataByID()
+	}
+
 	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		return nil, err
@@ -398,9 +425,16 @@ func (am *Auth0Manager) UpdateUserAppMetadata(userID string, appMetadata AppMeta
 
 	res, err := am.httpClient.Do(req)
 	if err != nil {
+		if am.appMetrics != nil {
+			am.appMetrics.IDPMetrics().CountRequestError()
+		}
 		return err
 	}
 
+	if am.appMetrics != nil {
+		am.appMetrics.IDPMetrics().CountUpdateUserAppMetadata()
+	}
+
 	defer func() {
 		err = res.Body.Close()
 		if err != nil {
@@ -416,12 +450,13 @@ func (am *Auth0Manager) UpdateUserAppMetadata(userID string, appMetadata AppMeta
 }
 
 func buildCreateUserRequestPayload(email string, name string, accountID string) (string, error) {
+	invite := true
 	req := &createUserRequest{
 		Email: email,
 		Name:  name,
 		AppMeta: AppMetadata{
 			WTAccountID:     accountID,
-			WTPendingInvite: true,
+			WTPendingInvite: &invite,
 		},
 		Connection:  "Username-Password-Authentication",
 		Password:    GeneratePassword(8, 1, 1, 1),
@@ -502,6 +537,9 @@ func (am *Auth0Manager) GetAllAccounts() (map[string][]*UserData, error) {
 	jobResp, err := am.httpClient.Do(exportJobReq)
 	if err != nil {
 		log.Debugf("Couldn't get job response %v", err)
+		if am.appMetrics != nil {
+			am.appMetrics.IDPMetrics().CountRequestError()
+		}
 		return nil, err
 	}
 
@@ -512,6 +550,9 @@ func (am *Auth0Manager) GetAllAccounts() (map[string][]*UserData, error) {
 		}
 	}()
 	if jobResp.StatusCode != 200 {
+		if am.appMetrics != nil {
+			am.appMetrics.IDPMetrics().CountRequestStatusError()
+		}
 		return nil, fmt.Errorf("unable to update the appMetadata, statusCode %d", jobResp.StatusCode)
 	}
 
@@ -530,6 +571,9 @@ func (am *Auth0Manager) GetAllAccounts() (map[string][]*UserData, error) {
 	}
 
 	if exportJobResp.ID == "" {
+		if am.appMetrics != nil {
+			am.appMetrics.IDPMetrics().CountRequestStatusError()
+		}
 		return nil, fmt.Errorf("couldn't get an batch id status %d, %s, response body: %v", jobResp.StatusCode, jobResp.Status, exportJobResp)
 	}
 
@@ -556,12 +600,16 @@ func (am *Auth0Manager) GetUserByEmail(email string) ([]*UserData, error) {
 	if err != nil {
 		return nil, err
 	}
-	reqURL := am.authIssuer + "/api/v2/users-by-email?email=" + email
+	reqURL := am.authIssuer + "/api/v2/users-by-email?email=" + url.QueryEscape(email)
 	body, err := doGetReq(am.httpClient, reqURL, jwtToken.AccessToken)
 	if err != nil {
 		return nil, err
 	}
 
+	if am.appMetrics != nil {
+		am.appMetrics.IDPMetrics().CountGetUserByEmail()
+	}
+
 	userResp := []*UserData{}
 
 	err = am.helper.Unmarshal(body, &userResp)
@@ -585,9 +633,16 @@ func (am *Auth0Manager) CreateUser(email string, name string, accountID string)
 		return nil, err
 	}
 
+	if am.appMetrics != nil {
+		am.appMetrics.IDPMetrics().CountCreateUser()
+	}
+
 	resp, err := am.httpClient.Do(req)
 	if err != nil {
 		log.Debugf("Couldn't get job response %v", err)
+		if am.appMetrics != nil {
+			am.appMetrics.IDPMetrics().CountRequestError()
+		}
 		return nil, err
 	}
 
@@ -598,6 +653,9 @@ func (am *Auth0Manager) CreateUser(email string, name string, accountID string)
 		}
 	}()
 	if !(resp.StatusCode == 200 || resp.StatusCode == 201) {
+		if am.appMetrics != nil {
+			am.appMetrics.IDPMetrics().CountRequestStatusError()
+		}
 		return nil, fmt.Errorf("unable to create user, statusCode %d", resp.StatusCode)
 	}
 
@@ -698,7 +756,7 @@ func (am *Auth0Manager) downloadProfileExport(location string) (map[string][]*Us
 					Email: profile.Email,
 					AppMetadata: AppMetadata{
 						WTAccountID:     profile.AccountID,
-						WTPendingInvite: profile.PendingInvite,
+						WTPendingInvite: &profile.PendingInvite,
 					},
 				})
 		}
@@ -729,13 +787,12 @@ func doGetReq(client ManagerHTTPClient, url, accessToken string) ([]byte, error)
 			log.Errorf("error while closing body for url %s: %v", url, err)
 		}
 	}()
-	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("unable to get %s, statusCode %d", url, res.StatusCode)
-	}
-
 	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		return nil, err
 	}
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("unable to get %s, statusCode %d", url, res.StatusCode)
+	}
 	return body, nil
 }

management/server/idp/auth0_test.go

@@ -3,6 +3,7 @@ package idp
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/netbirdio/netbird/management/server/telemetry"
 	"github.com/stretchr/testify/require"
 	"io"
 	"net/http"
@@ -340,7 +341,7 @@ func TestAuth0_UpdateUserAppMetadata(t *testing.T) {
 	updateUserAppMetadataTestCase2 := updateUserAppMetadataTest{
 		name:            "Bad Status Code",
 		inputReqBody:    fmt.Sprintf("{\"access_token\":\"%s\",\"scope\":\"read:users\",\"expires_in\":%d,\"token_type\":\"Bearer\"}", token, exp),
-		expectedReqBody: fmt.Sprintf("{\"app_metadata\":{\"wt_account_id\":\"%s\",\"wt_pending_invite\":false}}", appMetadata.WTAccountID),
+		expectedReqBody: fmt.Sprintf("{\"app_metadata\":{\"wt_account_id\":\"%s\",\"wt_pending_invite\":null}}", appMetadata.WTAccountID),
 		appMetadata:     appMetadata,
 		statusCode:      400,
 		helper:          JsonParser{},
@@ -363,7 +364,7 @@ func TestAuth0_UpdateUserAppMetadata(t *testing.T) {
 	updateUserAppMetadataTestCase4 := updateUserAppMetadataTest{
 		name:                 "Good request",
 		inputReqBody:         fmt.Sprintf("{\"access_token\":\"%s\",\"scope\":\"read:users\",\"expires_in\":%d,\"token_type\":\"Bearer\"}", token, exp),
-		expectedReqBody:      fmt.Sprintf("{\"app_metadata\":{\"wt_account_id\":\"%s\",\"wt_pending_invite\":false}}", appMetadata.WTAccountID),
+		expectedReqBody:      fmt.Sprintf("{\"app_metadata\":{\"wt_account_id\":\"%s\",\"wt_pending_invite\":null}}", appMetadata.WTAccountID),
 		appMetadata:          appMetadata,
 		statusCode:           200,
 		helper:               JsonParser{},
@@ -371,7 +372,23 @@ func TestAuth0_UpdateUserAppMetadata(t *testing.T) {
 		assertErrFuncMessage: "shouldn't return error",
 	}
 
-	for _, testCase := range []updateUserAppMetadataTest{updateUserAppMetadataTestCase1, updateUserAppMetadataTestCase2, updateUserAppMetadataTestCase3, updateUserAppMetadataTestCase4} {
+	invite := true
+	updateUserAppMetadataTestCase5 := updateUserAppMetadataTest{
+		name:            "Update Pending Invite",
+		inputReqBody:    fmt.Sprintf("{\"access_token\":\"%s\",\"scope\":\"read:users\",\"expires_in\":%d,\"token_type\":\"Bearer\"}", token, exp),
+		expectedReqBody: fmt.Sprintf("{\"app_metadata\":{\"wt_account_id\":\"%s\",\"wt_pending_invite\":true}}", appMetadata.WTAccountID),
+		appMetadata: AppMetadata{
+			WTAccountID:     "ok",
+			WTPendingInvite: &invite,
+		},
+		statusCode:           200,
+		helper:               JsonParser{},
+		assertErrFunc:        assert.NoError,
+		assertErrFuncMessage: "shouldn't return error",
+	}
+
+	for _, testCase := range []updateUserAppMetadataTest{updateUserAppMetadataTestCase1, updateUserAppMetadataTestCase2,
+		updateUserAppMetadataTestCase3, updateUserAppMetadataTestCase4, updateUserAppMetadataTestCase5} {
 		t.Run(testCase.name, func(t *testing.T) {
 			jwtReqClient := mockHTTPClient{
 				resBody: testCase.inputReqBody,
@@ -459,7 +476,7 @@ func TestNewAuth0Manager(t *testing.T) {
 
 	for _, testCase := range []test{testCase1, testCase2, testCase3, testCase4} {
 		t.Run(testCase.name, func(t *testing.T) {
-			_, err := NewAuth0Manager(testCase.inputConfig)
+			_, err := NewAuth0Manager(testCase.inputConfig, &telemetry.MockAppMetrics{})
 			testCase.assertErrFunc(t, err, testCase.assertErrFuncMessage)
 		})
 	}

management/server/idp/idp.go

@@ -2,6 +2,7 @@ package idp
 
 import (
 	"fmt"
+	"github.com/netbirdio/netbird/management/server/telemetry"
 	"net/http"
 	"strings"
 	"time"
@@ -51,7 +52,7 @@ type AppMetadata struct {
 	// WTAccountID is a NetBird (previously Wiretrustee) account id to update in the IDP
 	// maps to wt_account_id when json.marshal
 	WTAccountID     string `json:"wt_account_id,omitempty"`
-	WTPendingInvite bool   `json:"wt_pending_invite"`
+	WTPendingInvite *bool  `json:"wt_pending_invite"`
 }
 
 // JWTToken a JWT object that holds information of a token
@@ -64,12 +65,12 @@ type JWTToken struct {
 }
 
 // NewManager returns a new idp manager based on the configuration that it receives
-func NewManager(config Config) (Manager, error) {
+func NewManager(config Config, appMetrics telemetry.AppMetrics) (Manager, error) {
 	switch strings.ToLower(config.ManagerType) {
 	case "none", "":
 		return nil, nil
 	case "auth0":
-		return NewAuth0Manager(config.Auth0ClientCredentials)
+		return NewAuth0Manager(config.Auth0ClientCredentials, appMetrics)
 	default:
 		return nil, fmt.Errorf("invalid manager type: %s", config.ManagerType)
 	}

management/server/management_proto_test.go

@@ -403,12 +403,12 @@ func startManagement(t *testing.T, port int, config *Config) (*grpc.Server, erro
 		return nil, err
 	}
 	peersUpdateManager := NewPeersUpdateManager()
-	accountManager, err := BuildManager(store, peersUpdateManager, nil)
+	accountManager, err := BuildManager(store, peersUpdateManager, nil, "")
 	if err != nil {
 		return nil, err
 	}
 	turnManager := NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
-	mgmtServer, err := NewServer(config, accountManager, peersUpdateManager, turnManager)
+	mgmtServer, err := NewServer(config, accountManager, peersUpdateManager, turnManager, nil)
 	if err != nil {
 		return nil, err
 	}

management/server/management_test.go

@@ -493,12 +493,12 @@ func startServer(config *server.Config) (*grpc.Server, net.Listener) {
 		log.Fatalf("failed creating a store: %s: %v", config.Datadir, err)
 	}
 	peersUpdateManager := server.NewPeersUpdateManager()
-	accountManager, err := server.BuildManager(store, peersUpdateManager, nil)
+	accountManager, err := server.BuildManager(store, peersUpdateManager, nil, "")
 	if err != nil {
 		log.Fatalf("failed creating a manager: %v", err)
 	}
 	turnManager := server.NewTimeBasedAuthSecretsManager(peersUpdateManager, config.TURNConfig)
-	mgmtServer, err := server.NewServer(config, accountManager, peersUpdateManager, turnManager)
+	mgmtServer, err := server.NewServer(config, accountManager, peersUpdateManager, turnManager, nil)
 	Expect(err).NotTo(HaveOccurred())
 	mgmtProto.RegisterManagementServiceServer(s, mgmtServer)
 	go func() {

management/server/metrics/selfhosted.go

@@ -17,7 +17,7 @@ import (
 const (
 	// PayloadEvent identifies an event type
 	PayloadEvent = "self-hosted stats"
-	// payloadEndpoint metrics endpoint to send anonymous data
+	// payloadEndpoint metrics defaultEndpoint to send anonymous data
 	payloadEndpoint = "https://metrics.netbird.io"
 	// defaultPushInterval default interval to push metrics
 	defaultPushInterval = 24 * time.Hour

management/server/nameserver_test.go

@@ -865,7 +865,7 @@ func createNSManager(t *testing.T) (*DefaultAccountManager, error) {
 	if err != nil {
 		return nil, err
 	}
-	return BuildManager(store, NewPeersUpdateManager(), nil)
+	return BuildManager(store, NewPeersUpdateManager(), nil, "")
 }
 
 func createNSStore(t *testing.T) (Store, error) {

management/server/route_test.go

@@ -778,7 +778,7 @@ func createRouterManager(t *testing.T) (*DefaultAccountManager, error) {
 	if err != nil {
 		return nil, err
 	}
-	return BuildManager(store, NewPeersUpdateManager(), nil)
+	return BuildManager(store, NewPeersUpdateManager(), nil, "")
 }
 
 func createRouterStore(t *testing.T) (Store, error) {

management/server/telemetry/app_metrics.go

@@ -0,0 +1,181 @@
+package telemetry
+
+import (
+	"context"
+	"fmt"
+	"github.com/gorilla/mux"
+	prometheus2 "github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	metric2 "go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/sdk/metric"
+	"net"
+	"net/http"
+	"reflect"
+)
+
+const defaultEndpoint = "/metrics"
+
+// MockAppMetrics mocks the AppMetrics interface
+type MockAppMetrics struct {
+	GetMeterFunc       func() metric2.Meter
+	CloseFunc          func() error
+	ExposeFunc         func(port int, endpoint string) error
+	IDPMetricsFunc     func() *IDPMetrics
+	HTTPMiddlewareFunc func() *HTTPMiddleware
+	GRPCMetricsFunc    func() *GRPCMetrics
+}
+
+// GetMeter mocks the GetMeter function of the AppMetrics interface
+func (mock *MockAppMetrics) GetMeter() metric2.Meter {
+	if mock.GetMeterFunc != nil {
+		return mock.GetMeterFunc()
+	}
+	return nil
+}
+
+// Close mocks the Close function of the AppMetrics interface
+func (mock *MockAppMetrics) Close() error {
+	if mock.CloseFunc != nil {
+		return mock.CloseFunc()
+	}
+	return fmt.Errorf("unimplemented")
+}
+
+// Expose mocks the Expose function of the AppMetrics interface
+func (mock *MockAppMetrics) Expose(port int, endpoint string) error {
+	if mock.ExposeFunc != nil {
+		return mock.ExposeFunc(port, endpoint)
+	}
+	return fmt.Errorf("unimplemented")
+}
+
+// IDPMetrics mocks the IDPMetrics function of the IDPMetrics interface
+func (mock *MockAppMetrics) IDPMetrics() *IDPMetrics {
+	if mock.IDPMetricsFunc != nil {
+		return mock.IDPMetricsFunc()
+	}
+	return nil
+}
+
+// HTTPMiddleware mocks the HTTPMiddleware function of the IDPMetrics interface
+func (mock *MockAppMetrics) HTTPMiddleware() *HTTPMiddleware {
+	if mock.HTTPMiddlewareFunc != nil {
+		return mock.HTTPMiddlewareFunc()
+	}
+	return nil
+}
+
+// GRPCMetrics mocks the GRPCMetrics function of the IDPMetrics interface
+func (mock *MockAppMetrics) GRPCMetrics() *GRPCMetrics {
+	if mock.GRPCMetricsFunc != nil {
+		return mock.GRPCMetricsFunc()
+	}
+	return nil
+}
+
+// AppMetrics is metrics interface
+type AppMetrics interface {
+	GetMeter() metric2.Meter
+	Close() error
+	Expose(port int, endpoint string) error
+	IDPMetrics() *IDPMetrics
+	HTTPMiddleware() *HTTPMiddleware
+	GRPCMetrics() *GRPCMetrics
+}
+
+// defaultAppMetrics are core application metrics based on OpenTelemetry https://opentelemetry.io/
+type defaultAppMetrics struct {
+	// Meter can be used by different application parts to create counters and measure things
+	Meter          metric2.Meter
+	listener       net.Listener
+	ctx            context.Context
+	idpMetrics     *IDPMetrics
+	httpMiddleware *HTTPMiddleware
+	grpcMetrics    *GRPCMetrics
+}
+
+// IDPMetrics returns metrics for the idp package
+func (appMetrics *defaultAppMetrics) IDPMetrics() *IDPMetrics {
+	return appMetrics.idpMetrics
+}
+
+// HTTPMiddleware returns metrics for the http api package
+func (appMetrics *defaultAppMetrics) HTTPMiddleware() *HTTPMiddleware {
+	return appMetrics.httpMiddleware
+}
+
+// GRPCMetrics returns metrics for the gRPC api
+func (appMetrics *defaultAppMetrics) GRPCMetrics() *GRPCMetrics {
+	return appMetrics.grpcMetrics
+}
+
+// Close stop application metrics HTTP handler and closes listener.
+func (appMetrics *defaultAppMetrics) Close() error {
+	if appMetrics.listener == nil {
+		return nil
+	}
+	return appMetrics.listener.Close()
+}
+
+// Expose metrics on a given port and endpoint. If endpoint is empty a defaultEndpoint one will be used.
+// Exposes metrics in the Prometheus format https://prometheus.io/
+func (appMetrics *defaultAppMetrics) Expose(port int, endpoint string) error {
+	if endpoint == "" {
+		endpoint = defaultEndpoint
+	}
+	rootRouter := mux.NewRouter()
+	rootRouter.Handle(endpoint, promhttp.HandlerFor(
+		prometheus2.DefaultGatherer,
+		promhttp.HandlerOpts{EnableOpenMetrics: true}))
+	listener, err := net.Listen("tcp4", fmt.Sprintf(":%d", port))
+	if err != nil {
+		return err
+	}
+	appMetrics.listener = listener
+	go func() {
+		err := http.Serve(listener, rootRouter)
+		if err != nil {
+			return
+		}
+	}()
+
+	log.Infof("enabled application metrics and exposing on http://%s", listener.Addr().String())
+
+	return nil
+}
+
+// GetMeter returns metrics meter that can be used to add various counters
+func (appMetrics *defaultAppMetrics) GetMeter() metric2.Meter {
+	return appMetrics.Meter
+}
+
+// NewDefaultAppMetrics and expose them via defaultEndpoint on a given HTTP port
+func NewDefaultAppMetrics(ctx context.Context) (AppMetrics, error) {
+	exporter, err := prometheus.New()
+	if err != nil {
+		return nil, err
+	}
+
+	provider := metric.NewMeterProvider(metric.WithReader(exporter))
+	pkg := reflect.TypeOf(defaultEndpoint).PkgPath()
+	meter := provider.Meter(pkg)
+
+	idpMetrics, err := NewIDPMetrics(ctx, meter)
+	if err != nil {
+		return nil, err
+	}
+
+	middleware, err := NewMetricsMiddleware(ctx, meter)
+	if err != nil {
+		return nil, err
+	}
+	grpcMetrics, err := NewGRPCMetrics(ctx, meter)
+	if err != nil {
+		return nil, err
+	}
+
+	return &defaultAppMetrics{Meter: meter, ctx: ctx, idpMetrics: idpMetrics, httpMiddleware: middleware,
+		grpcMetrics: grpcMetrics}, nil
+}

management/server/telemetry/grpc_metrics.go

@@ -0,0 +1,76 @@
+package telemetry
+
+import (
+	"context"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/metric/instrument"
+	"go.opentelemetry.io/otel/metric/instrument/asyncint64"
+	"go.opentelemetry.io/otel/metric/instrument/syncint64"
+)
+
+// GRPCMetrics are gRPC server metrics
+type GRPCMetrics struct {
+	meter                 metric.Meter
+	syncRequestsCounter   syncint64.Counter
+	loginRequestsCounter  syncint64.Counter
+	getKeyRequestsCounter syncint64.Counter
+	activeStreamsGauge    asyncint64.Gauge
+	ctx                   context.Context
+}
+
+// NewGRPCMetrics creates new GRPCMetrics struct and registers common metrics of the gRPC server
+func NewGRPCMetrics(ctx context.Context, meter metric.Meter) (*GRPCMetrics, error) {
+	syncRequestsCounter, err := meter.SyncInt64().Counter("management.grpc.sync.request.counter", instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	loginRequestsCounter, err := meter.SyncInt64().Counter("management.grpc.login.request.counter", instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	getKeyRequestsCounter, err := meter.SyncInt64().Counter("management.grpc.key.request.counter", instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+
+	activeStreamsGauge, err := meter.AsyncInt64().Gauge("management.grpc.connected.streams", instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+
+	return &GRPCMetrics{
+		meter:                 meter,
+		syncRequestsCounter:   syncRequestsCounter,
+		loginRequestsCounter:  loginRequestsCounter,
+		getKeyRequestsCounter: getKeyRequestsCounter,
+		activeStreamsGauge:    activeStreamsGauge,
+		ctx:                   ctx,
+	}, err
+}
+
+// CountSyncRequest counts the number of gRPC sync requests coming to the gRPC API
+func (grpcMetrics *GRPCMetrics) CountSyncRequest() {
+	grpcMetrics.syncRequestsCounter.Add(grpcMetrics.ctx, 1)
+}
+
+// CountGetKeyRequest counts the number of gRPC get server key requests coming to the gRPC API
+func (grpcMetrics *GRPCMetrics) CountGetKeyRequest() {
+	grpcMetrics.getKeyRequestsCounter.Add(grpcMetrics.ctx, 1)
+}
+
+// CountLoginRequest counts the number of gRPC login requests coming to the gRPC API
+func (grpcMetrics *GRPCMetrics) CountLoginRequest() {
+	grpcMetrics.loginRequestsCounter.Add(grpcMetrics.ctx, 1)
+}
+
+// RegisterConnectedStreams registers a function that collects number of active streams and feeds it to the metrics gauge.
+func (grpcMetrics *GRPCMetrics) RegisterConnectedStreams(producer func() int64) error {
+	return grpcMetrics.meter.RegisterCallback(
+		[]instrument.Asynchronous{
+			grpcMetrics.activeStreamsGauge,
+		},
+		func(ctx context.Context) {
+			grpcMetrics.activeStreamsGauge.Observe(ctx, producer())
+		},
+	)
+}

management/server/telemetry/http_api_metrics.go

@@ -0,0 +1,176 @@
+package telemetry
+
+import (
+	"context"
+	"fmt"
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/metric/instrument"
+	"go.opentelemetry.io/otel/metric/instrument/syncint64"
+	"hash/fnv"
+	"net/http"
+	"strings"
+)
+
+const (
+	httpRequestCounterPrefix  = "management.http.request.counter"
+	httpResponseCounterPrefix = "management.http.response.counter"
+)
+
+// WrappedResponseWriter is a wrapper for http.ResponseWriter that allows the
+// written HTTP status code to be captured for metrics reporting or logging purposes.
+type WrappedResponseWriter struct {
+	http.ResponseWriter
+	status      int
+	wroteHeader bool
+}
+
+// WrapResponseWriter wraps original http.ResponseWriter
+func WrapResponseWriter(w http.ResponseWriter) *WrappedResponseWriter {
+	return &WrappedResponseWriter{ResponseWriter: w}
+}
+
+// Status returns response status
+func (rw *WrappedResponseWriter) Status() int {
+	return rw.status
+}
+
+// WriteHeader wraps http.ResponseWriter.WriteHeader method
+func (rw *WrappedResponseWriter) WriteHeader(code int) {
+	if rw.wroteHeader {
+		return
+	}
+
+	rw.status = code
+	rw.ResponseWriter.WriteHeader(code)
+	rw.wroteHeader = true
+}
+
+// HTTPMiddleware handler used to collect metrics of every request/response coming to the API.
+// Also adds request tracing (logging).
+type HTTPMiddleware struct {
+	meter metric.Meter
+	ctx   context.Context
+	// defaultEndpoint & method
+	httpRequestCounters map[string]syncint64.Counter
+	// defaultEndpoint & method & status code
+	httpResponseCounters map[string]syncint64.Counter
+	// all HTTP requests
+	totalHTTPRequestsCounter syncint64.Counter
+	// all HTTP responses
+	totalHTTPResponseCounter syncint64.Counter
+	// all HTTP responses by status code
+	totalHTTPResponseCodeCounters map[int]syncint64.Counter
+}
+
+// AddHTTPRequestResponseCounter adds a new meter for an HTTP defaultEndpoint and Method (GET, POST, etc)
+// Creates one request counter and multiple response counters (one per http response status code).
+func (m *HTTPMiddleware) AddHTTPRequestResponseCounter(endpoint string, method string) error {
+	meterKey := getRequestCounterKey(endpoint, method)
+	httpReqCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
+	if err != nil {
+		return err
+	}
+	m.httpRequestCounters[meterKey] = httpReqCounter
+	respCodes := []int{200, 204, 400, 401, 403, 404, 500, 502, 503}
+	for _, code := range respCodes {
+		meterKey = getResponseCounterKey(endpoint, method, code)
+		httpRespCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
+		if err != nil {
+			return err
+		}
+		m.httpResponseCounters[meterKey] = httpRespCounter
+
+		meterKey = fmt.Sprintf("%s_%d_total", httpResponseCounterPrefix, code)
+		totalHTTPResponseCodeCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
+		if err != nil {
+			return err
+		}
+		m.totalHTTPResponseCodeCounters[code] = totalHTTPResponseCodeCounter
+	}
+
+	return nil
+}
+
+// NewMetricsMiddleware creates a new HTTPMiddleware
+func NewMetricsMiddleware(ctx context.Context, meter metric.Meter) (*HTTPMiddleware, error) {
+
+	totalHTTPRequestsCounter, err := meter.SyncInt64().Counter(
+		fmt.Sprintf("%s_total", httpRequestCounterPrefix),
+		instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	totalHTTPResponseCounter, err := meter.SyncInt64().Counter(
+		fmt.Sprintf("%s_total", httpResponseCounterPrefix),
+		instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	return &HTTPMiddleware{
+			ctx:                           ctx,
+			httpRequestCounters:           map[string]syncint64.Counter{},
+			httpResponseCounters:          map[string]syncint64.Counter{},
+			totalHTTPResponseCodeCounters: map[int]syncint64.Counter{},
+			meter:                         meter,
+			totalHTTPRequestsCounter:      totalHTTPRequestsCounter,
+			totalHTTPResponseCounter:      totalHTTPResponseCounter,
+		},
+		nil
+}
+
+func getRequestCounterKey(endpoint, method string) string {
+	return fmt.Sprintf("%s%s_%s", httpRequestCounterPrefix,
+		strings.ReplaceAll(endpoint, "/", "_"), method)
+}
+
+func getResponseCounterKey(endpoint, method string, status int) string {
+	return fmt.Sprintf("%s%s_%s_%d", httpResponseCounterPrefix,
+		strings.ReplaceAll(endpoint, "/", "_"), method, status)
+}
+
+// Handler logs every request and response and adds the, to metrics.
+func (m *HTTPMiddleware) Handler(h http.Handler) http.Handler {
+	fn := func(rw http.ResponseWriter, r *http.Request) {
+		traceID := hash(fmt.Sprintf("%v", r))
+		log.Tracef("HTTP request %v: %v %v", traceID, r.Method, r.URL)
+
+		metricKey := getRequestCounterKey(r.URL.Path, r.Method)
+
+		if c, ok := m.httpRequestCounters[metricKey]; ok {
+			c.Add(m.ctx, 1)
+		}
+		m.totalHTTPRequestsCounter.Add(m.ctx, 1)
+
+		w := WrapResponseWriter(rw)
+
+		h.ServeHTTP(w, r)
+
+		if w.Status() > 399 {
+			log.Errorf("HTTP response %v: %v %v status %v", traceID, r.Method, r.URL, w.Status())
+		} else {
+			log.Tracef("HTTP response %v: %v %v status %v", traceID, r.Method, r.URL, w.Status())
+		}
+
+		metricKey = getResponseCounterKey(r.URL.Path, r.Method, w.Status())
+		if c, ok := m.httpResponseCounters[metricKey]; ok {
+			c.Add(m.ctx, 1)
+		}
+
+		m.totalHTTPResponseCounter.Add(m.ctx, 1)
+		if c, ok := m.totalHTTPResponseCodeCounters[w.Status()]; ok {
+			c.Add(m.ctx, 1)
+		}
+	}
+
+	return http.HandlerFunc(fn)
+}
+
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	_, err := h.Write([]byte(s))
+	if err != nil {
+		panic(err)
+	}
+	return h.Sum32()
+}

management/server/telemetry/idp_metrics.go

@@ -0,0 +1,119 @@
+package telemetry
+
+import (
+	"context"
+	"go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/metric/instrument"
+	"go.opentelemetry.io/otel/metric/instrument/syncint64"
+)
+
+// IDPMetrics is common IdP metrics
+type IDPMetrics struct {
+	metaUpdateCounter          syncint64.Counter
+	getUserByEmailCounter      syncint64.Counter
+	getAllAccountsCounter      syncint64.Counter
+	createUserCounter          syncint64.Counter
+	getAccountCounter          syncint64.Counter
+	getUserByIDCounter         syncint64.Counter
+	authenticateRequestCounter syncint64.Counter
+	requestErrorCounter        syncint64.Counter
+	requestStatusErrorCounter  syncint64.Counter
+	ctx                        context.Context
+}
+
+// NewIDPMetrics creates new IDPMetrics struct and registers common
+func NewIDPMetrics(ctx context.Context, meter metric.Meter) (*IDPMetrics, error) {
+	metaUpdateCounter, err := meter.SyncInt64().Counter("management.idp.update.user.meta.counter", instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	getUserByEmailCounter, err := meter.SyncInt64().Counter("management.idp.get.user.by.email.counter", instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	getAllAccountsCounter, err := meter.SyncInt64().Counter("management.idp.get.accounts.counter", instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	createUserCounter, err := meter.SyncInt64().Counter("management.idp.create.user.counter", instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	getAccountCounter, err := meter.SyncInt64().Counter("management.idp.get.account.counter", instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	getUserByIDCounter, err := meter.SyncInt64().Counter("management.idp.get.user.by.id.counter", instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	authenticateRequestCounter, err := meter.SyncInt64().Counter("management.idp.authenticate.request.counter", instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	requestErrorCounter, err := meter.SyncInt64().Counter("management.idp.request.error.counter", instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	requestStatusErrorCounter, err := meter.SyncInt64().Counter("management.idp.request.status.error.counter", instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+
+	return &IDPMetrics{
+		metaUpdateCounter:          metaUpdateCounter,
+		getUserByEmailCounter:      getUserByEmailCounter,
+		getAllAccountsCounter:      getAllAccountsCounter,
+		createUserCounter:          createUserCounter,
+		getAccountCounter:          getAccountCounter,
+		getUserByIDCounter:         getUserByIDCounter,
+		authenticateRequestCounter: authenticateRequestCounter,
+		requestErrorCounter:        requestErrorCounter,
+		requestStatusErrorCounter:  requestStatusErrorCounter,
+		ctx:                        ctx}, nil
+}
+
+// CountUpdateUserAppMetadata ...
+func (idpMetrics *IDPMetrics) CountUpdateUserAppMetadata() {
+	idpMetrics.metaUpdateCounter.Add(idpMetrics.ctx, 1)
+}
+
+// CountGetUserByEmail ...
+func (idpMetrics *IDPMetrics) CountGetUserByEmail() {
+	idpMetrics.getUserByEmailCounter.Add(idpMetrics.ctx, 1)
+}
+
+// CountCreateUser ...
+func (idpMetrics *IDPMetrics) CountCreateUser() {
+	idpMetrics.createUserCounter.Add(idpMetrics.ctx, 1)
+}
+
+// CountGetAllAccounts ...
+func (idpMetrics *IDPMetrics) CountGetAllAccounts() {
+	idpMetrics.getAllAccountsCounter.Add(idpMetrics.ctx, 1)
+}
+
+// CountGetAccount ...
+func (idpMetrics *IDPMetrics) CountGetAccount() {
+	idpMetrics.getAccountCounter.Add(idpMetrics.ctx, 1)
+}
+
+// CountGetUserDataByID ...
+func (idpMetrics *IDPMetrics) CountGetUserDataByID() {
+	idpMetrics.getUserByIDCounter.Add(idpMetrics.ctx, 1)
+}
+
+// CountAuthenticate ...
+func (idpMetrics *IDPMetrics) CountAuthenticate() {
+	idpMetrics.authenticateRequestCounter.Add(idpMetrics.ctx, 1)
+}
+
+// CountRequestError counts number of error that happened when doing http request (httpClient.Do)
+func (idpMetrics *IDPMetrics) CountRequestError() {
+	idpMetrics.requestErrorCounter.Add(idpMetrics.ctx, 1)
+}
+
+// CountRequestStatusError counts number of responses that came from IdP with non success status code
+func (idpMetrics *IDPMetrics) CountRequestStatusError() {
+	idpMetrics.requestStatusErrorCounter.Add(idpMetrics.ctx, 1)
+}

management/server/user.go

@@ -68,7 +68,7 @@ func (u *User) toUserInfo(userData *idp.UserData) (*UserInfo, error) {
 	}
 
 	userStatus := UserStatusActive
-	if userData.AppMetadata.WTPendingInvite {
+	if userData.AppMetadata.WTPendingInvite != nil && *userData.AppMetadata.WTPendingInvite {
 		userStatus = UserStatusInvited
 	}
 

util/file.go

@@ -3,7 +3,6 @@ package util
 import (
 	"encoding/json"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 )
@@ -24,7 +23,7 @@ func WriteJson(file string, obj interface{}) error {
 		return err
 	}
 
-	tempFile, err := ioutil.TempFile(configDir, ".*"+configFileName)
+	tempFile, err := os.CreateTemp(configDir, ".*"+configFileName)
 	if err != nil {
 		return err
 	}
@@ -43,7 +42,7 @@ func WriteJson(file string, obj interface{}) error {
 		}
 	}()
 
-	err = ioutil.WriteFile(tempFileName, bs, 0600)
+	err = os.WriteFile(tempFileName, bs, 0600)
 	if err != nil {
 		return err
 	}
@@ -65,7 +64,7 @@ func ReadJson(file string, res interface{}) (interface{}, error) {
 	}
 	defer f.Close()
 
-	bs, err := ioutil.ReadAll(f)
+	bs, err := io.ReadAll(f)
 	if err != nil {
 		return nil, err
 	}