Add Open-telemetry support (#517)

This PR brings open-telemetry metrics to the Management service. The Management service exposes new HTTP endpoint /metrics on 8081 port by default. The port can be changed by specifying --metrics-port PORT flag when starting the service.
2026-04-18 08:16:39 +00:00 · 2022-10-21 16:24:13 +02:00
parent 08ddf04c5f
commit 4f1f0df7d2
7 changed files with 296 additions and 58 deletions
--- a/management/server/http/handler.go
+++ b/management/server/http/handler.go
@@ -1,15 +1,18 @@
 package http

 import (
+	"context"
 	"github.com/gorilla/mux"
 	s "github.com/netbirdio/netbird/management/server"
 	"github.com/netbirdio/netbird/management/server/http/middleware"
 	"github.com/rs/cors"
+	"go.opentelemetry.io/otel/metric"
 	"net/http"
 )

 // APIHandler creates the Management service HTTP API handler registering all the available endpoints.
-func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience string, authKeysLocation string) (http.Handler, error) {
+func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience string, authKeysLocation string,
+	meter metric.Meter) (http.Handler, error) {
 	jwtMiddleware, err := middleware.NewJwtMiddleware(
 		authIssuer,
 		authAudience,
@@ -25,8 +28,14 @@ func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience
 		authAudience,
 		accountManager.IsUserAdmin)

-	apiHandler := mux.NewRouter()
-	apiHandler.Use(corsMiddleware.Handler, jwtMiddleware.Handler, acMiddleware.Handler)
+	rootRouter := mux.NewRouter()
+	metrics, err := middleware.NewMetricsMiddleware(context.Background(), meter)
+	if err != nil {
+		return nil, err
+	}
+
+	apiHandler := rootRouter.PathPrefix("/api").Subrouter()
+	apiHandler.Use(metrics.Handler, corsMiddleware.Handler, jwtMiddleware.Handler, acMiddleware.Handler)

 	groupsHandler := NewGroups(accountManager, authAudience)
 	rulesHandler := NewRules(accountManager, authAudience)
@@ -36,46 +45,67 @@ func APIHandler(accountManager s.AccountManager, authIssuer string, authAudience
 	routesHandler := NewRoutes(accountManager, authAudience)
 	nameserversHandler := NewNameservers(accountManager, authAudience)

-	apiHandler.HandleFunc("/api/peers", peersHandler.GetPeers).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/peers/{id}", peersHandler.HandlePeer).
+	apiHandler.HandleFunc("/peers", peersHandler.GetPeers).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/peers/{id}", peersHandler.HandlePeer).
 		Methods("GET", "PUT", "DELETE", "OPTIONS")
-	apiHandler.HandleFunc("/api/users", userHandler.GetUsers).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/users/{id}", userHandler.UpdateUser).Methods("PUT", "OPTIONS")
-	apiHandler.HandleFunc("/api/users", userHandler.CreateUserHandler).Methods("POST", "OPTIONS")
+	apiHandler.HandleFunc("/users", userHandler.GetUsers).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/users/{id}", userHandler.UpdateUser).Methods("PUT", "OPTIONS")
+	apiHandler.HandleFunc("/users", userHandler.CreateUserHandler).Methods("POST", "OPTIONS")

-	apiHandler.HandleFunc("/api/setup-keys", keysHandler.GetAllSetupKeysHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/setup-keys", keysHandler.CreateSetupKeyHandler).Methods("POST", "OPTIONS")
-	apiHandler.HandleFunc("/api/setup-keys/{id}", keysHandler.GetSetupKeyHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/setup-keys/{id}", keysHandler.UpdateSetupKeyHandler).Methods("PUT", "OPTIONS")
+	apiHandler.HandleFunc("/setup-keys", keysHandler.GetAllSetupKeysHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/setup-keys", keysHandler.CreateSetupKeyHandler).Methods("POST", "OPTIONS")
+	apiHandler.HandleFunc("/setup-keys/{id}", keysHandler.GetSetupKeyHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/setup-keys/{id}", keysHandler.UpdateSetupKeyHandler).Methods("PUT", "OPTIONS")

-	apiHandler.HandleFunc("/api/rules", rulesHandler.GetAllRulesHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/rules", rulesHandler.CreateRuleHandler).Methods("POST", "OPTIONS")
-	apiHandler.HandleFunc("/api/rules/{id}", rulesHandler.UpdateRuleHandler).Methods("PUT", "OPTIONS")
-	apiHandler.HandleFunc("/api/rules/{id}", rulesHandler.PatchRuleHandler).Methods("PATCH", "OPTIONS")
-	apiHandler.HandleFunc("/api/rules/{id}", rulesHandler.GetRuleHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/rules/{id}", rulesHandler.DeleteRuleHandler).Methods("DELETE", "OPTIONS")
+	apiHandler.HandleFunc("/rules", rulesHandler.GetAllRulesHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/rules", rulesHandler.CreateRuleHandler).Methods("POST", "OPTIONS")
+	apiHandler.HandleFunc("/rules/{id}", rulesHandler.UpdateRuleHandler).Methods("PUT", "OPTIONS")
+	apiHandler.HandleFunc("/rules/{id}", rulesHandler.PatchRuleHandler).Methods("PATCH", "OPTIONS")
+	apiHandler.HandleFunc("/rules/{id}", rulesHandler.GetRuleHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/rules/{id}", rulesHandler.DeleteRuleHandler).Methods("DELETE", "OPTIONS")

-	apiHandler.HandleFunc("/api/groups", groupsHandler.GetAllGroupsHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/groups", groupsHandler.CreateGroupHandler).Methods("POST", "OPTIONS")
-	apiHandler.HandleFunc("/api/groups/{id}", groupsHandler.UpdateGroupHandler).Methods("PUT", "OPTIONS")
-	apiHandler.HandleFunc("/api/groups/{id}", groupsHandler.PatchGroupHandler).Methods("PATCH", "OPTIONS")
-	apiHandler.HandleFunc("/api/groups/{id}", groupsHandler.GetGroupHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/groups/{id}", groupsHandler.DeleteGroupHandler).Methods("DELETE", "OPTIONS")
+	apiHandler.HandleFunc("/groups", groupsHandler.GetAllGroupsHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/groups", groupsHandler.CreateGroupHandler).Methods("POST", "OPTIONS")
+	apiHandler.HandleFunc("/groups/{id}", groupsHandler.UpdateGroupHandler).Methods("PUT", "OPTIONS")
+	apiHandler.HandleFunc("/groups/{id}", groupsHandler.PatchGroupHandler).Methods("PATCH", "OPTIONS")
+	apiHandler.HandleFunc("/groups/{id}", groupsHandler.GetGroupHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/groups/{id}", groupsHandler.DeleteGroupHandler).Methods("DELETE", "OPTIONS")

-	apiHandler.HandleFunc("/api/routes", routesHandler.GetAllRoutesHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/routes", routesHandler.CreateRouteHandler).Methods("POST", "OPTIONS")
-	apiHandler.HandleFunc("/api/routes/{id}", routesHandler.UpdateRouteHandler).Methods("PUT", "OPTIONS")
-	apiHandler.HandleFunc("/api/routes/{id}", routesHandler.PatchRouteHandler).Methods("PATCH", "OPTIONS")
-	apiHandler.HandleFunc("/api/routes/{id}", routesHandler.GetRouteHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/routes/{id}", routesHandler.DeleteRouteHandler).Methods("DELETE", "OPTIONS")
+	apiHandler.HandleFunc("/routes", routesHandler.GetAllRoutesHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/routes", routesHandler.CreateRouteHandler).Methods("POST", "OPTIONS")
+	apiHandler.HandleFunc("/routes/{id}", routesHandler.UpdateRouteHandler).Methods("PUT", "OPTIONS")
+	apiHandler.HandleFunc("/routes/{id}", routesHandler.PatchRouteHandler).Methods("PATCH", "OPTIONS")
+	apiHandler.HandleFunc("/routes/{id}", routesHandler.GetRouteHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/routes/{id}", routesHandler.DeleteRouteHandler).Methods("DELETE", "OPTIONS")

-	apiHandler.HandleFunc("/api/dns/nameservers", nameserversHandler.GetAllNameserversHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/dns/nameservers", nameserversHandler.CreateNameserverGroupHandler).Methods("POST", "OPTIONS")
-	apiHandler.HandleFunc("/api/dns/nameservers/{id}", nameserversHandler.UpdateNameserverGroupHandler).Methods("PUT", "OPTIONS")
-	apiHandler.HandleFunc("/api/dns/nameservers/{id}", nameserversHandler.PatchNameserverGroupHandler).Methods("PATCH", "OPTIONS")
-	apiHandler.HandleFunc("/api/dns/nameservers/{id}", nameserversHandler.GetNameserverGroupHandler).Methods("GET", "OPTIONS")
-	apiHandler.HandleFunc("/api/dns/nameservers/{id}", nameserversHandler.DeleteNameserverGroupHandler).Methods("DELETE", "OPTIONS")
+	apiHandler.HandleFunc("/dns/nameservers", nameserversHandler.GetAllNameserversHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/dns/nameservers", nameserversHandler.CreateNameserverGroupHandler).Methods("POST", "OPTIONS")
+	apiHandler.HandleFunc("/dns/nameservers/{id}", nameserversHandler.UpdateNameserverGroupHandler).Methods("PUT", "OPTIONS")
+	apiHandler.HandleFunc("/dns/nameservers/{id}", nameserversHandler.PatchNameserverGroupHandler).Methods("PATCH", "OPTIONS")
+	apiHandler.HandleFunc("/dns/nameservers/{id}", nameserversHandler.GetNameserverGroupHandler).Methods("GET", "OPTIONS")
+	apiHandler.HandleFunc("/dns/nameservers/{id}", nameserversHandler.DeleteNameserverGroupHandler).Methods("DELETE", "OPTIONS")

-	return apiHandler, nil
+	err = apiHandler.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error {
+		methods, err := route.GetMethods()
+		if err != nil {
+			return err
+		}
+		for _, method := range methods {
+			template, err := route.GetPathTemplate()
+			if err != nil {
+				return err
+			}
+			err = metrics.AddHTTPRequestResponseCounter(template, method)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return rootRouter, nil

 }
--- a/management/server/http/middleware/jwtmiddleware.go
+++ b/management/server/http/middleware/jwtmiddleware.go
@@ -186,7 +186,7 @@ func (m *JWTMiddleware) CheckJWTFromRequest(w http.ResponseWriter, r *http.Reque

 	validatedToken, err := m.ValidateAndParse(token)
 	if err != nil {
-		m.Options.ErrorHandler(w, r, "The token isn't valid")
+		m.Options.ErrorHandler(w, r, err.Error())
 		return err
 	}

--- a/management/server/http/middleware/metrics.go
+++ b/management/server/http/middleware/metrics.go
@@ -0,0 +1,139 @@
+package middleware
+
+import (
+	"context"
+	"fmt"
+	log "github.com/sirupsen/logrus"
+	metric2 "go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/metric/instrument"
+	"go.opentelemetry.io/otel/metric/instrument/syncint64"
+	"hash/fnv"
+	"net/http"
+	"strings"
+)
+
+const (
+	httpRequestCounterPrefix  = "management.http.request.counter"
+	httpResponseCounterPrefix = "management.http.response.counter"
+)
+
+// WrappedResponseWriter is a wrapper for http.ResponseWriter that allows the
+// written HTTP status code to be captured for metrics reporting or logging purposes.
+type WrappedResponseWriter struct {
+	http.ResponseWriter
+	status      int
+	wroteHeader bool
+}
+
+// WrapResponseWriter wraps original http.ResponseWriter
+func WrapResponseWriter(w http.ResponseWriter) *WrappedResponseWriter {
+	return &WrappedResponseWriter{ResponseWriter: w}
+}
+
+// Status returns response status
+func (rw *WrappedResponseWriter) Status() int {
+	return rw.status
+}
+
+// WriteHeader wraps http.ResponseWriter.WriteHeader method
+func (rw *WrappedResponseWriter) WriteHeader(code int) {
+	if rw.wroteHeader {
+		return
+	}
+
+	rw.status = code
+	rw.ResponseWriter.WriteHeader(code)
+	rw.wroteHeader = true
+}
+
+// MetricsMiddleware handler used to collect metrics of every request/response coming to the API.
+// Also adds request tracing (logging).
+type MetricsMiddleware struct {
+	meter                metric2.Meter
+	ctx                  context.Context
+	httpRequestCounters  map[string]syncint64.Counter
+	httpResponseCounters map[string]syncint64.Counter
+}
+
+// AddHTTPRequestResponseCounter adds a new meter for an HTTP endpoint and Method (GET, POST, etc)
+// Creates one request counter and multiple response counters (one per http response status code).
+func (m *MetricsMiddleware) AddHTTPRequestResponseCounter(endpoint string, method string) error {
+	meterKey := getRequestCounterKey(endpoint, method)
+	httpReqCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
+	if err != nil {
+		return err
+	}
+	m.httpRequestCounters[meterKey] = httpReqCounter
+	respCodes := []int{200, 204, 400, 401, 403, 500, 502, 503}
+	for _, code := range respCodes {
+		meterKey = getResponseCounterKey(endpoint, method, code)
+		httpRespCounter, err := m.meter.SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
+		if err != nil {
+			return err
+		}
+		m.httpResponseCounters[meterKey] = httpRespCounter
+	}
+
+	return nil
+}
+
+// NewMetricsMiddleware creates a new MetricsMiddleware
+func NewMetricsMiddleware(ctx context.Context, meter metric2.Meter) (*MetricsMiddleware, error) {
+	return &MetricsMiddleware{
+			ctx:                  ctx,
+			httpRequestCounters:  map[string]syncint64.Counter{},
+			httpResponseCounters: map[string]syncint64.Counter{},
+			meter:                meter,
+		},
+		nil
+}
+
+func getRequestCounterKey(endpoint, method string) string {
+	return fmt.Sprintf("%s%s_%s", httpRequestCounterPrefix,
+		strings.ReplaceAll(endpoint, "/", "_"), method)
+}
+
+func getResponseCounterKey(endpoint, method string, status int) string {
+	return fmt.Sprintf("%s%s_%s_%d", httpResponseCounterPrefix,
+		strings.ReplaceAll(endpoint, "/", "_"), method, status)
+}
+
+// Handler logs every request and response and adds the, to metrics.
+func (m *MetricsMiddleware) Handler(h http.Handler) http.Handler {
+	fn := func(rw http.ResponseWriter, r *http.Request) {
+		traceID := hash(fmt.Sprintf("%v", r))
+		log.Tracef("HTTP request %v: %v %v", traceID, r.Method, r.URL)
+
+		metricKey := getRequestCounterKey(r.URL.Path, r.Method)
+
+		if c, ok := m.httpRequestCounters[metricKey]; ok {
+			c.Add(m.ctx, 1)
+		}
+
+		w := WrapResponseWriter(rw)
+
+		h.ServeHTTP(w, r)
+
+		if w.Status() > 399 {
+			log.Errorf("HTTP response %v: %v %v status %v", traceID, r.Method, r.URL, w.Status())
+		} else {
+			log.Tracef("HTTP response %v: %v %v status %v", traceID, r.Method, r.URL, w.Status())
+		}
+
+		metricKey = getResponseCounterKey(r.URL.Path, r.Method, w.Status())
+		if c, ok := m.httpResponseCounters[metricKey]; ok {
+			c.Add(m.ctx, 1)
+		}
+	}
+
+	return http.HandlerFunc(fn)
+}
+
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	_, err := h.Write([]byte(s))
+	if err != nil {
+		panic(err)
+	}
+	return h.Sum32()
+}