diff --git a/management/cmd/management.go b/management/cmd/management.go index e46493d97..ea8e73c88 100644 --- a/management/cmd/management.go +++ b/management/cmd/management.go @@ -10,6 +10,7 @@ import ( "github.com/google/uuid" httpapi "github.com/netbirdio/netbird/management/server/http" "github.com/netbirdio/netbird/management/server/metrics" + "github.com/netbirdio/netbird/management/server/telemetry" "golang.org/x/crypto/acme/autocert" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" @@ -115,9 +116,18 @@ var ( } peersUpdateManager := server.NewPeersUpdateManager() + appMetrics, err := telemetry.NewDefaultAppMetrics(cmd.Context()) + if err != nil { + return err + } + err = appMetrics.Expose(mgmtMetricsPort, "/metrics") + if err != nil { + return err + } + var idpManager idp.Manager if config.IdpManagerConfig != nil { - idpManager, err = idp.NewManager(*config.IdpManagerConfig) + idpManager, err = idp.NewManager(*config.IdpManagerConfig, appMetrics) if err != nil { return fmt.Errorf("failed retrieving a new idp manager with err: %v", err) } @@ -155,16 +165,8 @@ var ( gRPCOpts = append(gRPCOpts, grpc.Creds(transportCredentials)) tlsEnabled = true } - appMetrics, err := metrics.NewDefaultAppMetrics(cmd.Context()) - if err != nil { - return err - } - err = appMetrics.Expose(mgmtMetricsPort, "/metrics") - if err != nil { - return err - } - httpAPIHandler, err := httpapi.APIHandler(cmd.Context(), accountManager, config.HttpConfig.AuthIssuer, + httpAPIHandler, err := httpapi.APIHandler(accountManager, config.HttpConfig.AuthIssuer, config.HttpConfig.AuthAudience, config.HttpConfig.AuthKeysLocation, appMetrics) if err != nil { return fmt.Errorf("failed creating HTTP API handler: %v", err) diff --git a/management/server/http/handler.go b/management/server/http/handler.go index 16a21a154..1f85dd995 100644 --- a/management/server/http/handler.go +++ b/management/server/http/handler.go @@ -1,18 +1,17 @@ package http import ( - "context" "github.com/gorilla/mux" s "github.com/netbirdio/netbird/management/server" "github.com/netbirdio/netbird/management/server/http/middleware" - "github.com/netbirdio/netbird/management/server/metrics" + "github.com/netbirdio/netbird/management/server/telemetry" "github.com/rs/cors" "net/http" ) // APIHandler creates the Management service HTTP API handler registering all the available endpoints. -func APIHandler(ctx context.Context, accountManager s.AccountManager, authIssuer string, authAudience string, authKeysLocation string, - appMetrics metrics.AppMetrics) (http.Handler, error) { +func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience string, authKeysLocation string, + appMetrics telemetry.AppMetrics) (http.Handler, error) { jwtMiddleware, err := middleware.NewJwtMiddleware( authIssuer, authAudience, @@ -29,10 +28,7 @@ func APIHandler(ctx context.Context, accountManager s.AccountManager, authIssuer accountManager.IsUserAdmin) rootRouter := mux.NewRouter() - metricsMiddleware, err := metrics.NewMetricsMiddleware(ctx, appMetrics) - if err != nil { - return nil, err - } + metricsMiddleware := appMetrics.HTTPMiddleware() apiHandler := rootRouter.PathPrefix("/api").Subrouter() apiHandler.Use(metricsMiddleware.Handler, corsMiddleware.Handler, jwtMiddleware.Handler, acMiddleware.Handler) diff --git a/management/server/idp/auth0.go b/management/server/idp/auth0.go index d90e0ade2..a19117326 100644 --- a/management/server/idp/auth0.go +++ b/management/server/idp/auth0.go @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/netbirdio/netbird/management/server/telemetry" "io" "net/http" "net/url" @@ -24,6 +25,7 @@ type Auth0Manager struct { httpClient ManagerHTTPClient credentials ManagerCredentials helper ManagerHelper + appMetrics telemetry.AppMetrics } // Auth0ClientConfig auth0 manager client configurations @@ -51,6 +53,7 @@ type Auth0Credentials struct { httpClient ManagerHTTPClient jwtToken JWTToken mux sync.Mutex + appMetrics telemetry.AppMetrics } // createUserRequest is a user create request @@ -106,7 +109,7 @@ type auth0Profile struct { } // NewAuth0Manager creates a new instance of the Auth0Manager -func NewAuth0Manager(config Auth0ClientConfig) (*Auth0Manager, error) { +func NewAuth0Manager(config Auth0ClientConfig, appMetrics telemetry.AppMetrics) (*Auth0Manager, error) { httpTransport := http.DefaultTransport.(*http.Transport).Clone() httpTransport.MaxIdleConns = 5 @@ -134,12 +137,15 @@ func NewAuth0Manager(config Auth0ClientConfig) (*Auth0Manager, error) { clientConfig: config, httpClient: httpClient, helper: helper, + appMetrics: appMetrics, } + return &Auth0Manager{ authIssuer: config.AuthIssuer, credentials: credentials, httpClient: httpClient, helper: helper, + appMetrics: appMetrics, }, nil } @@ -170,6 +176,9 @@ func (c *Auth0Credentials) requestJWTToken() (*http.Response, error) { res, err = c.httpClient.Do(req) if err != nil { + if c.appMetrics != nil { + c.appMetrics.IDPMetrics().CountRequestError() + } return res, err } @@ -214,6 +223,10 @@ func (c *Auth0Credentials) Authenticate() (JWTToken, error) { c.mux.Lock() defer c.mux.Unlock() + if c.appMetrics != nil { + c.appMetrics.IDPMetrics().CountAuthenticate() + } + // If jwtToken has an expires time and we have enough time to do a request return immediately if c.jwtStillValid() { return c.jwtToken, nil @@ -287,9 +300,16 @@ func (am *Auth0Manager) GetAccount(accountID string) ([]*UserData, error) { res, err := am.httpClient.Do(req) if err != nil { + if am.appMetrics != nil { + am.appMetrics.IDPMetrics().CountRequestError() + } return nil, err } + if am.appMetrics != nil { + am.appMetrics.IDPMetrics().CountGetAccount() + } + body, err := io.ReadAll(res.Body) if err != nil { return nil, err @@ -342,9 +362,16 @@ func (am *Auth0Manager) GetUserDataByID(userID string, appMetadata AppMetadata) res, err := am.httpClient.Do(req) if err != nil { + if am.appMetrics != nil { + am.appMetrics.IDPMetrics().CountRequestError() + } return nil, err } + if am.appMetrics != nil { + am.appMetrics.IDPMetrics().CountGetUserDataByID() + } + body, err := io.ReadAll(res.Body) if err != nil { return nil, err @@ -398,9 +425,16 @@ func (am *Auth0Manager) UpdateUserAppMetadata(userID string, appMetadata AppMeta res, err := am.httpClient.Do(req) if err != nil { + if am.appMetrics != nil { + am.appMetrics.IDPMetrics().CountRequestError() + } return err } + if am.appMetrics != nil { + am.appMetrics.IDPMetrics().CountUpdateUserAppMetadata() + } + defer func() { err = res.Body.Close() if err != nil { @@ -503,6 +537,9 @@ func (am *Auth0Manager) GetAllAccounts() (map[string][]*UserData, error) { jobResp, err := am.httpClient.Do(exportJobReq) if err != nil { log.Debugf("Couldn't get job response %v", err) + if am.appMetrics != nil { + am.appMetrics.IDPMetrics().CountRequestError() + } return nil, err } @@ -513,6 +550,9 @@ func (am *Auth0Manager) GetAllAccounts() (map[string][]*UserData, error) { } }() if jobResp.StatusCode != 200 { + if am.appMetrics != nil { + am.appMetrics.IDPMetrics().CountRequestStatusError() + } return nil, fmt.Errorf("unable to update the appMetadata, statusCode %d", jobResp.StatusCode) } @@ -531,6 +571,9 @@ func (am *Auth0Manager) GetAllAccounts() (map[string][]*UserData, error) { } if exportJobResp.ID == "" { + if am.appMetrics != nil { + am.appMetrics.IDPMetrics().CountRequestStatusError() + } return nil, fmt.Errorf("couldn't get an batch id status %d, %s, response body: %v", jobResp.StatusCode, jobResp.Status, exportJobResp) } @@ -563,6 +606,10 @@ func (am *Auth0Manager) GetUserByEmail(email string) ([]*UserData, error) { return nil, err } + if am.appMetrics != nil { + am.appMetrics.IDPMetrics().CountGetUserByEmail() + } + userResp := []*UserData{} err = am.helper.Unmarshal(body, &userResp) @@ -586,9 +633,16 @@ func (am *Auth0Manager) CreateUser(email string, name string, accountID string) return nil, err } + if am.appMetrics != nil { + am.appMetrics.IDPMetrics().CountCreateUser() + } + resp, err := am.httpClient.Do(req) if err != nil { log.Debugf("Couldn't get job response %v", err) + if am.appMetrics != nil { + am.appMetrics.IDPMetrics().CountRequestError() + } return nil, err } @@ -599,6 +653,9 @@ func (am *Auth0Manager) CreateUser(email string, name string, accountID string) } }() if !(resp.StatusCode == 200 || resp.StatusCode == 201) { + if am.appMetrics != nil { + am.appMetrics.IDPMetrics().CountRequestStatusError() + } return nil, fmt.Errorf("unable to create user, statusCode %d", resp.StatusCode) } diff --git a/management/server/idp/auth0_test.go b/management/server/idp/auth0_test.go index f05fa76fc..58a6df8f6 100644 --- a/management/server/idp/auth0_test.go +++ b/management/server/idp/auth0_test.go @@ -3,6 +3,7 @@ package idp import ( "encoding/json" "fmt" + "github.com/netbirdio/netbird/management/server/telemetry" "github.com/stretchr/testify/require" "io" "net/http" @@ -475,7 +476,7 @@ func TestNewAuth0Manager(t *testing.T) { for _, testCase := range []test{testCase1, testCase2, testCase3, testCase4} { t.Run(testCase.name, func(t *testing.T) { - _, err := NewAuth0Manager(testCase.inputConfig) + _, err := NewAuth0Manager(testCase.inputConfig, &telemetry.MockAppMetrics{}) testCase.assertErrFunc(t, err, testCase.assertErrFuncMessage) }) } diff --git a/management/server/idp/idp.go b/management/server/idp/idp.go index 5d9cf67a4..724a3541d 100644 --- a/management/server/idp/idp.go +++ b/management/server/idp/idp.go @@ -2,6 +2,7 @@ package idp import ( "fmt" + "github.com/netbirdio/netbird/management/server/telemetry" "net/http" "strings" "time" @@ -64,12 +65,12 @@ type JWTToken struct { } // NewManager returns a new idp manager based on the configuration that it receives -func NewManager(config Config) (Manager, error) { +func NewManager(config Config, appMetrics telemetry.AppMetrics) (Manager, error) { switch strings.ToLower(config.ManagerType) { case "none", "": return nil, nil case "auth0": - return NewAuth0Manager(config.Auth0ClientCredentials) + return NewAuth0Manager(config.Auth0ClientCredentials, appMetrics) default: return nil, fmt.Errorf("invalid manager type: %s", config.ManagerType) } diff --git a/management/server/metrics/app.go b/management/server/telemetry/app_metrics.go similarity index 51% rename from management/server/metrics/app.go rename to management/server/telemetry/app_metrics.go index 454c86e04..54bde57cb 100644 --- a/management/server/metrics/app.go +++ b/management/server/telemetry/app_metrics.go @@ -1,4 +1,4 @@ -package metrics +package telemetry import ( "context" @@ -17,19 +17,82 @@ import ( const defaultEndpoint = "/metrics" +// MockAppMetrics mocks the AppMetrics interface +type MockAppMetrics struct { + GetMeterFunc func() metric2.Meter + CloseFunc func() error + ExposeFunc func(port int, endpoint string) error + IDPMetricsFunc func() *IDPMetrics + HTTPMiddlewareFunc func() *HTTPMiddleware +} + +// GetMeter mocks the GetMeter function of the AppMetrics interface +func (mock *MockAppMetrics) GetMeter() metric2.Meter { + if mock.GetMeterFunc != nil { + return mock.GetMeterFunc() + } + return nil +} + +// Close mocks the Close function of the AppMetrics interface +func (mock *MockAppMetrics) Close() error { + if mock.CloseFunc != nil { + return mock.CloseFunc() + } + return fmt.Errorf("unimplemented") +} + +// Expose mocks the Expose function of the AppMetrics interface +func (mock *MockAppMetrics) Expose(port int, endpoint string) error { + if mock.ExposeFunc != nil { + return mock.ExposeFunc(port, endpoint) + } + return fmt.Errorf("unimplemented") +} + +// IDPMetrics mocks the IDPMetrics function of the IDPMetrics interface +func (mock *MockAppMetrics) IDPMetrics() *IDPMetrics { + if mock.IDPMetricsFunc != nil { + return mock.IDPMetricsFunc() + } + return nil +} + +// HTTPMiddleware mocks the HTTPMiddleware function of the IDPMetrics interface +func (mock *MockAppMetrics) HTTPMiddleware() *HTTPMiddleware { + if mock.HTTPMiddlewareFunc != nil { + return mock.HTTPMiddlewareFunc() + } + return nil +} + // AppMetrics is metrics interface type AppMetrics interface { GetMeter() metric2.Meter Close() error Expose(port int, endpoint string) error + IDPMetrics() *IDPMetrics + HTTPMiddleware() *HTTPMiddleware } // defaultAppMetrics are core application metrics based on OpenTelemetry https://opentelemetry.io/ type defaultAppMetrics struct { // Meter can be used by different application parts to create counters and measure things - Meter metric2.Meter - listener net.Listener - ctx context.Context + Meter metric2.Meter + listener net.Listener + ctx context.Context + idpMetrics *IDPMetrics + httpMiddleware *HTTPMiddleware +} + +// IDPMetrics returns metrics for the idp package +func (appMetrics *defaultAppMetrics) IDPMetrics() *IDPMetrics { + return appMetrics.idpMetrics +} + +// HTTPMiddleware returns metrics for the http api package +func (appMetrics *defaultAppMetrics) HTTPMiddleware() *HTTPMiddleware { + return appMetrics.httpMiddleware } // Close stop application metrics HTTP handler and closes listener. @@ -83,5 +146,15 @@ func NewDefaultAppMetrics(ctx context.Context) (AppMetrics, error) { pkg := reflect.TypeOf(defaultEndpoint).PkgPath() meter := provider.Meter(pkg) - return &defaultAppMetrics{Meter: meter, ctx: ctx}, nil + idpMetrics, err := NewIDPMetrics(ctx, meter) + if err != nil { + return nil, err + } + + middleware, err := NewMetricsMiddleware(ctx, meter) + if err != nil { + return nil, err + } + + return &defaultAppMetrics{Meter: meter, ctx: ctx, idpMetrics: idpMetrics, httpMiddleware: middleware}, nil } diff --git a/management/server/metrics/middleware.go b/management/server/telemetry/http_api_metrics.go similarity index 87% rename from management/server/metrics/middleware.go rename to management/server/telemetry/http_api_metrics.go index a347c7bd9..652ac92a9 100644 --- a/management/server/metrics/middleware.go +++ b/management/server/telemetry/http_api_metrics.go @@ -1,9 +1,10 @@ -package metrics +package telemetry import ( "context" "fmt" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/metric/instrument" "go.opentelemetry.io/otel/metric/instrument/syncint64" "hash/fnv" @@ -48,8 +49,8 @@ func (rw *WrappedResponseWriter) WriteHeader(code int) { // HTTPMiddleware handler used to collect metrics of every request/response coming to the API. // Also adds request tracing (logging). type HTTPMiddleware struct { - appMetrics AppMetrics - ctx context.Context + meter metric.Meter + ctx context.Context // defaultEndpoint & method httpRequestCounters map[string]syncint64.Counter // defaultEndpoint & method & status code @@ -66,7 +67,7 @@ type HTTPMiddleware struct { // Creates one request counter and multiple response counters (one per http response status code). func (m *HTTPMiddleware) AddHTTPRequestResponseCounter(endpoint string, method string) error { meterKey := getRequestCounterKey(endpoint, method) - httpReqCounter, err := m.appMetrics.GetMeter().SyncInt64().Counter(meterKey, instrument.WithUnit("1")) + httpReqCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1")) if err != nil { return err } @@ -74,14 +75,14 @@ func (m *HTTPMiddleware) AddHTTPRequestResponseCounter(endpoint string, method s respCodes := []int{200, 204, 400, 401, 403, 404, 500, 502, 503} for _, code := range respCodes { meterKey = getResponseCounterKey(endpoint, method, code) - httpRespCounter, err := m.appMetrics.GetMeter().SyncInt64().Counter(meterKey, instrument.WithUnit("1")) + httpRespCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1")) if err != nil { return err } m.httpResponseCounters[meterKey] = httpRespCounter meterKey = fmt.Sprintf("%s_%d_total", httpResponseCounterPrefix, code) - totalHTTPResponseCodeCounter, err := m.appMetrics.GetMeter().SyncInt64().Counter(meterKey, instrument.WithUnit("1")) + totalHTTPResponseCodeCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1")) if err != nil { return err } @@ -92,15 +93,15 @@ func (m *HTTPMiddleware) AddHTTPRequestResponseCounter(endpoint string, method s } // NewMetricsMiddleware creates a new HTTPMiddleware -func NewMetricsMiddleware(ctx context.Context, appMetrics AppMetrics) (*HTTPMiddleware, error) { +func NewMetricsMiddleware(ctx context.Context, meter metric.Meter) (*HTTPMiddleware, error) { - totalHTTPRequestsCounter, err := appMetrics.GetMeter().SyncInt64().Counter( + totalHTTPRequestsCounter, err := meter.SyncInt64().Counter( fmt.Sprintf("%s_total", httpRequestCounterPrefix), instrument.WithUnit("1")) if err != nil { return nil, err } - totalHTTPResponseCounter, err := appMetrics.GetMeter().SyncInt64().Counter( + totalHTTPResponseCounter, err := meter.SyncInt64().Counter( fmt.Sprintf("%s_total", httpResponseCounterPrefix), instrument.WithUnit("1")) if err != nil { @@ -111,7 +112,7 @@ func NewMetricsMiddleware(ctx context.Context, appMetrics AppMetrics) (*HTTPMidd httpRequestCounters: map[string]syncint64.Counter{}, httpResponseCounters: map[string]syncint64.Counter{}, totalHTTPResponseCodeCounters: map[int]syncint64.Counter{}, - appMetrics: appMetrics, + meter: meter, totalHTTPRequestsCounter: totalHTTPRequestsCounter, totalHTTPResponseCounter: totalHTTPResponseCounter, }, diff --git a/management/server/telemetry/idp_metrics.go b/management/server/telemetry/idp_metrics.go new file mode 100644 index 000000000..67a1d9e85 --- /dev/null +++ b/management/server/telemetry/idp_metrics.go @@ -0,0 +1,119 @@ +package telemetry + +import ( + "context" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/metric/instrument" + "go.opentelemetry.io/otel/metric/instrument/syncint64" +) + +// IDPMetrics is common IdP metrics +type IDPMetrics struct { + metaUpdateCounter syncint64.Counter + getUserByEmailCounter syncint64.Counter + getAllAccountsCounter syncint64.Counter + createUserCounter syncint64.Counter + getAccountCounter syncint64.Counter + getUserByIDCounter syncint64.Counter + authenticateRequestCounter syncint64.Counter + requestErrorCounter syncint64.Counter + requestStatusErrorCounter syncint64.Counter + ctx context.Context +} + +// NewIDPMetrics creates new IDPMetrics struct and registers common +func NewIDPMetrics(ctx context.Context, meter metric.Meter) (*IDPMetrics, error) { + metaUpdateCounter, err := meter.SyncInt64().Counter("management.idp.update.user.meta.counter", instrument.WithUnit("1")) + if err != nil { + return nil, err + } + getUserByEmailCounter, err := meter.SyncInt64().Counter("management.idp.get.user.by.email.counter", instrument.WithUnit("1")) + if err != nil { + return nil, err + } + getAllAccountsCounter, err := meter.SyncInt64().Counter("management.idp.get.accounts.counter", instrument.WithUnit("1")) + if err != nil { + return nil, err + } + createUserCounter, err := meter.SyncInt64().Counter("management.idp.create.user.counter", instrument.WithUnit("1")) + if err != nil { + return nil, err + } + getAccountCounter, err := meter.SyncInt64().Counter("management.idp.get.account.counter", instrument.WithUnit("1")) + if err != nil { + return nil, err + } + getUserByIDCounter, err := meter.SyncInt64().Counter("management.idp.get.user.by.id.counter", instrument.WithUnit("1")) + if err != nil { + return nil, err + } + authenticateRequestCounter, err := meter.SyncInt64().Counter("management.idp.authenticate.request.counter", instrument.WithUnit("1")) + if err != nil { + return nil, err + } + requestErrorCounter, err := meter.SyncInt64().Counter("management.idp.request.error.counter", instrument.WithUnit("1")) + if err != nil { + return nil, err + } + requestStatusErrorCounter, err := meter.SyncInt64().Counter("management.idp.request.status.error.counter", instrument.WithUnit("1")) + if err != nil { + return nil, err + } + + return &IDPMetrics{ + metaUpdateCounter: metaUpdateCounter, + getUserByEmailCounter: getUserByEmailCounter, + getAllAccountsCounter: getAllAccountsCounter, + createUserCounter: createUserCounter, + getAccountCounter: getAccountCounter, + getUserByIDCounter: getUserByIDCounter, + authenticateRequestCounter: authenticateRequestCounter, + requestErrorCounter: requestErrorCounter, + requestStatusErrorCounter: requestStatusErrorCounter, + ctx: ctx}, nil +} + +// CountUpdateUserAppMetadata ... +func (idpMetrics *IDPMetrics) CountUpdateUserAppMetadata() { + idpMetrics.metaUpdateCounter.Add(idpMetrics.ctx, 1) +} + +// CountGetUserByEmail ... +func (idpMetrics *IDPMetrics) CountGetUserByEmail() { + idpMetrics.getUserByEmailCounter.Add(idpMetrics.ctx, 1) +} + +// CountCreateUser ... +func (idpMetrics *IDPMetrics) CountCreateUser() { + idpMetrics.createUserCounter.Add(idpMetrics.ctx, 1) +} + +// CountGetAllAccounts ... +func (idpMetrics *IDPMetrics) CountGetAllAccounts() { + idpMetrics.getAllAccountsCounter.Add(idpMetrics.ctx, 1) +} + +// CountGetAccount ... +func (idpMetrics *IDPMetrics) CountGetAccount() { + idpMetrics.getAccountCounter.Add(idpMetrics.ctx, 1) +} + +// CountGetUserDataByID ... +func (idpMetrics *IDPMetrics) CountGetUserDataByID() { + idpMetrics.getUserByIDCounter.Add(idpMetrics.ctx, 1) +} + +// CountAuthenticate ... +func (idpMetrics *IDPMetrics) CountAuthenticate() { + idpMetrics.authenticateRequestCounter.Add(idpMetrics.ctx, 1) +} + +// CountRequestError counts number of error that happened when doing http request (httpClient.Do) +func (idpMetrics *IDPMetrics) CountRequestError() { + idpMetrics.requestErrorCounter.Add(idpMetrics.ctx, 1) +} + +// CountRequestStatusError counts number of responses that came from IdP with non success status code +func (idpMetrics *IDPMetrics) CountRequestStatusError() { + idpMetrics.requestStatusErrorCounter.Add(idpMetrics.ctx, 1) +}