mirror of
https://github.com/netbirdio/netbird.git
synced 2026-10-21 04:49:10 +02:00
Add IdP metrics (#521)
This commit is contained in:
Misha Bragin
@@ -0,0 +1,160 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/gorilla/mux"
|
||||
prometheus2 "github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.opentelemetry.io/otel/exporters/prometheus"
|
||||
metric2 "go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/sdk/metric"
|
||||
"net"
|
||||
"net/http"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
const defaultEndpoint = "/metrics"
|
||||
|
||||
// MockAppMetrics mocks the AppMetrics interface
|
||||
type MockAppMetrics struct {
|
||||
GetMeterFunc func() metric2.Meter
|
||||
CloseFunc func() error
|
||||
ExposeFunc func(port int, endpoint string) error
|
||||
IDPMetricsFunc func() *IDPMetrics
|
||||
HTTPMiddlewareFunc func() *HTTPMiddleware
|
||||
}
|
||||
|
||||
// GetMeter mocks the GetMeter function of the AppMetrics interface
|
||||
func (mock *MockAppMetrics) GetMeter() metric2.Meter {
|
||||
if mock.GetMeterFunc != nil {
|
||||
return mock.GetMeterFunc()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close mocks the Close function of the AppMetrics interface
|
||||
func (mock *MockAppMetrics) Close() error {
|
||||
if mock.CloseFunc != nil {
|
||||
return mock.CloseFunc()
|
||||
}
|
||||
return fmt.Errorf("unimplemented")
|
||||
}
|
||||
|
||||
// Expose mocks the Expose function of the AppMetrics interface
|
||||
func (mock *MockAppMetrics) Expose(port int, endpoint string) error {
|
||||
if mock.ExposeFunc != nil {
|
||||
return mock.ExposeFunc(port, endpoint)
|
||||
}
|
||||
return fmt.Errorf("unimplemented")
|
||||
}
|
||||
|
||||
// IDPMetrics mocks the IDPMetrics function of the IDPMetrics interface
|
||||
func (mock *MockAppMetrics) IDPMetrics() *IDPMetrics {
|
||||
if mock.IDPMetricsFunc != nil {
|
||||
return mock.IDPMetricsFunc()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HTTPMiddleware mocks the HTTPMiddleware function of the IDPMetrics interface
|
||||
func (mock *MockAppMetrics) HTTPMiddleware() *HTTPMiddleware {
|
||||
if mock.HTTPMiddlewareFunc != nil {
|
||||
return mock.HTTPMiddlewareFunc()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AppMetrics is metrics interface
|
||||
type AppMetrics interface {
|
||||
GetMeter() metric2.Meter
|
||||
Close() error
|
||||
Expose(port int, endpoint string) error
|
||||
IDPMetrics() *IDPMetrics
|
||||
HTTPMiddleware() *HTTPMiddleware
|
||||
}
|
||||
|
||||
// defaultAppMetrics are core application metrics based on OpenTelemetry https://opentelemetry.io/
|
||||
type defaultAppMetrics struct {
|
||||
// Meter can be used by different application parts to create counters and measure things
|
||||
Meter metric2.Meter
|
||||
listener net.Listener
|
||||
ctx context.Context
|
||||
idpMetrics *IDPMetrics
|
||||
httpMiddleware *HTTPMiddleware
|
||||
}
|
||||
|
||||
// IDPMetrics returns metrics for the idp package
|
||||
func (appMetrics *defaultAppMetrics) IDPMetrics() *IDPMetrics {
|
||||
return appMetrics.idpMetrics
|
||||
}
|
||||
|
||||
// HTTPMiddleware returns metrics for the http api package
|
||||
func (appMetrics *defaultAppMetrics) HTTPMiddleware() *HTTPMiddleware {
|
||||
return appMetrics.httpMiddleware
|
||||
}
|
||||
|
||||
// Close stop application metrics HTTP handler and closes listener.
|
||||
func (appMetrics *defaultAppMetrics) Close() error {
|
||||
if appMetrics.listener == nil {
|
||||
return nil
|
||||
}
|
||||
return appMetrics.listener.Close()
|
||||
}
|
||||
|
||||
// Expose metrics on a given port and endpoint. If endpoint is empty a defaultEndpoint one will be used.
|
||||
// Exposes metrics in the Prometheus format https://prometheus.io/
|
||||
func (appMetrics *defaultAppMetrics) Expose(port int, endpoint string) error {
|
||||
if endpoint == "" {
|
||||
endpoint = defaultEndpoint
|
||||
}
|
||||
rootRouter := mux.NewRouter()
|
||||
rootRouter.Handle(endpoint, promhttp.HandlerFor(
|
||||
prometheus2.DefaultGatherer,
|
||||
promhttp.HandlerOpts{EnableOpenMetrics: true}))
|
||||
listener, err := net.Listen("tcp4", fmt.Sprintf(":%d", port))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
appMetrics.listener = listener
|
||||
go func() {
|
||||
err := http.Serve(listener, rootRouter)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
log.Infof("enabled application metrics and exposing on http://%s", listener.Addr().String())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetMeter returns metrics meter that can be used to add various counters
|
||||
func (appMetrics *defaultAppMetrics) GetMeter() metric2.Meter {
|
||||
return appMetrics.Meter
|
||||
}
|
||||
|
||||
// NewDefaultAppMetrics and expose them via defaultEndpoint on a given HTTP port
|
||||
func NewDefaultAppMetrics(ctx context.Context) (AppMetrics, error) {
|
||||
exporter, err := prometheus.New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
provider := metric.NewMeterProvider(metric.WithReader(exporter))
|
||||
pkg := reflect.TypeOf(defaultEndpoint).PkgPath()
|
||||
meter := provider.Meter(pkg)
|
||||
|
||||
idpMetrics, err := NewIDPMetrics(ctx, meter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
middleware, err := NewMetricsMiddleware(ctx, meter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &defaultAppMetrics{Meter: meter, ctx: ctx, idpMetrics: idpMetrics, httpMiddleware: middleware}, nil
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/metric/instrument"
|
||||
"go.opentelemetry.io/otel/metric/instrument/syncint64"
|
||||
"hash/fnv"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
httpRequestCounterPrefix = "management.http.request.counter"
|
||||
httpResponseCounterPrefix = "management.http.response.counter"
|
||||
)
|
||||
|
||||
// WrappedResponseWriter is a wrapper for http.ResponseWriter that allows the
|
||||
// written HTTP status code to be captured for metrics reporting or logging purposes.
|
||||
type WrappedResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
status int
|
||||
wroteHeader bool
|
||||
}
|
||||
|
||||
// WrapResponseWriter wraps original http.ResponseWriter
|
||||
func WrapResponseWriter(w http.ResponseWriter) *WrappedResponseWriter {
|
||||
return &WrappedResponseWriter{ResponseWriter: w}
|
||||
}
|
||||
|
||||
// Status returns response status
|
||||
func (rw *WrappedResponseWriter) Status() int {
|
||||
return rw.status
|
||||
}
|
||||
|
||||
// WriteHeader wraps http.ResponseWriter.WriteHeader method
|
||||
func (rw *WrappedResponseWriter) WriteHeader(code int) {
|
||||
if rw.wroteHeader {
|
||||
return
|
||||
}
|
||||
|
||||
rw.status = code
|
||||
rw.ResponseWriter.WriteHeader(code)
|
||||
rw.wroteHeader = true
|
||||
}
|
||||
|
||||
// HTTPMiddleware handler used to collect metrics of every request/response coming to the API.
|
||||
// Also adds request tracing (logging).
|
||||
type HTTPMiddleware struct {
|
||||
meter metric.Meter
|
||||
ctx context.Context
|
||||
// defaultEndpoint & method
|
||||
httpRequestCounters map[string]syncint64.Counter
|
||||
// defaultEndpoint & method & status code
|
||||
httpResponseCounters map[string]syncint64.Counter
|
||||
// all HTTP requests
|
||||
totalHTTPRequestsCounter syncint64.Counter
|
||||
// all HTTP responses
|
||||
totalHTTPResponseCounter syncint64.Counter
|
||||
// all HTTP responses by status code
|
||||
totalHTTPResponseCodeCounters map[int]syncint64.Counter
|
||||
}
|
||||
|
||||
// AddHTTPRequestResponseCounter adds a new meter for an HTTP defaultEndpoint and Method (GET, POST, etc)
|
||||
// Creates one request counter and multiple response counters (one per http response status code).
|
||||
func (m *HTTPMiddleware) AddHTTPRequestResponseCounter(endpoint string, method string) error {
|
||||
meterKey := getRequestCounterKey(endpoint, method)
|
||||
httpReqCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.httpRequestCounters[meterKey] = httpReqCounter
|
||||
respCodes := []int{200, 204, 400, 401, 403, 404, 500, 502, 503}
|
||||
for _, code := range respCodes {
|
||||
meterKey = getResponseCounterKey(endpoint, method, code)
|
||||
httpRespCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.httpResponseCounters[meterKey] = httpRespCounter
|
||||
|
||||
meterKey = fmt.Sprintf("%s_%d_total", httpResponseCounterPrefix, code)
|
||||
totalHTTPResponseCodeCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.totalHTTPResponseCodeCounters[code] = totalHTTPResponseCodeCounter
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewMetricsMiddleware creates a new HTTPMiddleware
|
||||
func NewMetricsMiddleware(ctx context.Context, meter metric.Meter) (*HTTPMiddleware, error) {
|
||||
|
||||
totalHTTPRequestsCounter, err := meter.SyncInt64().Counter(
|
||||
fmt.Sprintf("%s_total", httpRequestCounterPrefix),
|
||||
instrument.WithUnit("1"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
totalHTTPResponseCounter, err := meter.SyncInt64().Counter(
|
||||
fmt.Sprintf("%s_total", httpResponseCounterPrefix),
|
||||
instrument.WithUnit("1"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &HTTPMiddleware{
|
||||
ctx: ctx,
|
||||
httpRequestCounters: map[string]syncint64.Counter{},
|
||||
httpResponseCounters: map[string]syncint64.Counter{},
|
||||
totalHTTPResponseCodeCounters: map[int]syncint64.Counter{},
|
||||
meter: meter,
|
||||
totalHTTPRequestsCounter: totalHTTPRequestsCounter,
|
||||
totalHTTPResponseCounter: totalHTTPResponseCounter,
|
||||
},
|
||||
nil
|
||||
}
|
||||
|
||||
func getRequestCounterKey(endpoint, method string) string {
|
||||
return fmt.Sprintf("%s%s_%s", httpRequestCounterPrefix,
|
||||
strings.ReplaceAll(endpoint, "/", "_"), method)
|
||||
}
|
||||
|
||||
func getResponseCounterKey(endpoint, method string, status int) string {
|
||||
return fmt.Sprintf("%s%s_%s_%d", httpResponseCounterPrefix,
|
||||
strings.ReplaceAll(endpoint, "/", "_"), method, status)
|
||||
}
|
||||
|
||||
// Handler logs every request and response and adds the, to metrics.
|
||||
func (m *HTTPMiddleware) Handler(h http.Handler) http.Handler {
|
||||
fn := func(rw http.ResponseWriter, r *http.Request) {
|
||||
traceID := hash(fmt.Sprintf("%v", r))
|
||||
log.Tracef("HTTP request %v: %v %v", traceID, r.Method, r.URL)
|
||||
|
||||
metricKey := getRequestCounterKey(r.URL.Path, r.Method)
|
||||
|
||||
if c, ok := m.httpRequestCounters[metricKey]; ok {
|
||||
c.Add(m.ctx, 1)
|
||||
}
|
||||
m.totalHTTPRequestsCounter.Add(m.ctx, 1)
|
||||
|
||||
w := WrapResponseWriter(rw)
|
||||
|
||||
h.ServeHTTP(w, r)
|
||||
|
||||
if w.Status() > 399 {
|
||||
log.Errorf("HTTP response %v: %v %v status %v", traceID, r.Method, r.URL, w.Status())
|
||||
} else {
|
||||
log.Tracef("HTTP response %v: %v %v status %v", traceID, r.Method, r.URL, w.Status())
|
||||
}
|
||||
|
||||
metricKey = getResponseCounterKey(r.URL.Path, r.Method, w.Status())
|
||||
if c, ok := m.httpResponseCounters[metricKey]; ok {
|
||||
c.Add(m.ctx, 1)
|
||||
}
|
||||
|
||||
m.totalHTTPResponseCounter.Add(m.ctx, 1)
|
||||
if c, ok := m.totalHTTPResponseCodeCounters[w.Status()]; ok {
|
||||
c.Add(m.ctx, 1)
|
||||
}
|
||||
}
|
||||
|
||||
return http.HandlerFunc(fn)
|
||||
}
|
||||
|
||||
func hash(s string) uint32 {
|
||||
h := fnv.New32a()
|
||||
_, err := h.Write([]byte(s))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return h.Sum32()
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/metric/instrument"
|
||||
"go.opentelemetry.io/otel/metric/instrument/syncint64"
|
||||
)
|
||||
|
||||
// IDPMetrics is common IdP metrics
|
||||
type IDPMetrics struct {
|
||||
metaUpdateCounter syncint64.Counter
|
||||
getUserByEmailCounter syncint64.Counter
|
||||
getAllAccountsCounter syncint64.Counter
|
||||
createUserCounter syncint64.Counter
|
||||
getAccountCounter syncint64.Counter
|
||||
getUserByIDCounter syncint64.Counter
|
||||
authenticateRequestCounter syncint64.Counter
|
||||
requestErrorCounter syncint64.Counter
|
||||
requestStatusErrorCounter syncint64.Counter
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
// NewIDPMetrics creates new IDPMetrics struct and registers common
|
||||
func NewIDPMetrics(ctx context.Context, meter metric.Meter) (*IDPMetrics, error) {
|
||||
metaUpdateCounter, err := meter.SyncInt64().Counter("management.idp.update.user.meta.counter", instrument.WithUnit("1"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
getUserByEmailCounter, err := meter.SyncInt64().Counter("management.idp.get.user.by.email.counter", instrument.WithUnit("1"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
getAllAccountsCounter, err := meter.SyncInt64().Counter("management.idp.get.accounts.counter", instrument.WithUnit("1"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
createUserCounter, err := meter.SyncInt64().Counter("management.idp.create.user.counter", instrument.WithUnit("1"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
getAccountCounter, err := meter.SyncInt64().Counter("management.idp.get.account.counter", instrument.WithUnit("1"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
getUserByIDCounter, err := meter.SyncInt64().Counter("management.idp.get.user.by.id.counter", instrument.WithUnit("1"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
authenticateRequestCounter, err := meter.SyncInt64().Counter("management.idp.authenticate.request.counter", instrument.WithUnit("1"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
requestErrorCounter, err := meter.SyncInt64().Counter("management.idp.request.error.counter", instrument.WithUnit("1"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
requestStatusErrorCounter, err := meter.SyncInt64().Counter("management.idp.request.status.error.counter", instrument.WithUnit("1"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &IDPMetrics{
|
||||
metaUpdateCounter: metaUpdateCounter,
|
||||
getUserByEmailCounter: getUserByEmailCounter,
|
||||
getAllAccountsCounter: getAllAccountsCounter,
|
||||
createUserCounter: createUserCounter,
|
||||
getAccountCounter: getAccountCounter,
|
||||
getUserByIDCounter: getUserByIDCounter,
|
||||
authenticateRequestCounter: authenticateRequestCounter,
|
||||
requestErrorCounter: requestErrorCounter,
|
||||
requestStatusErrorCounter: requestStatusErrorCounter,
|
||||
ctx: ctx}, nil
|
||||
}
|
||||
|
||||
// CountUpdateUserAppMetadata ...
|
||||
func (idpMetrics *IDPMetrics) CountUpdateUserAppMetadata() {
|
||||
idpMetrics.metaUpdateCounter.Add(idpMetrics.ctx, 1)
|
||||
}
|
||||
|
||||
// CountGetUserByEmail ...
|
||||
func (idpMetrics *IDPMetrics) CountGetUserByEmail() {
|
||||
idpMetrics.getUserByEmailCounter.Add(idpMetrics.ctx, 1)
|
||||
}
|
||||
|
||||
// CountCreateUser ...
|
||||
func (idpMetrics *IDPMetrics) CountCreateUser() {
|
||||
idpMetrics.createUserCounter.Add(idpMetrics.ctx, 1)
|
||||
}
|
||||
|
||||
// CountGetAllAccounts ...
|
||||
func (idpMetrics *IDPMetrics) CountGetAllAccounts() {
|
||||
idpMetrics.getAllAccountsCounter.Add(idpMetrics.ctx, 1)
|
||||
}
|
||||
|
||||
// CountGetAccount ...
|
||||
func (idpMetrics *IDPMetrics) CountGetAccount() {
|
||||
idpMetrics.getAccountCounter.Add(idpMetrics.ctx, 1)
|
||||
}
|
||||
|
||||
// CountGetUserDataByID ...
|
||||
func (idpMetrics *IDPMetrics) CountGetUserDataByID() {
|
||||
idpMetrics.getUserByIDCounter.Add(idpMetrics.ctx, 1)
|
||||
}
|
||||
|
||||
// CountAuthenticate ...
|
||||
func (idpMetrics *IDPMetrics) CountAuthenticate() {
|
||||
idpMetrics.authenticateRequestCounter.Add(idpMetrics.ctx, 1)
|
||||
}
|
||||
|
||||
// CountRequestError counts number of error that happened when doing http request (httpClient.Do)
|
||||
func (idpMetrics *IDPMetrics) CountRequestError() {
|
||||
idpMetrics.requestErrorCounter.Add(idpMetrics.ctx, 1)
|
||||
}
|
||||
|
||||
// CountRequestStatusError counts number of responses that came from IdP with non success status code
|
||||
func (idpMetrics *IDPMetrics) CountRequestStatusError() {
|
||||
idpMetrics.requestStatusErrorCounter.Add(idpMetrics.ctx, 1)
|
||||
}
|
||||
Reference in New Issue
Block a user