Extract app metrics to a separate struct (#520)

management/server/metrics/app.go

@@ -0,0 +1,87 @@
+package metrics
+
+import (
+	"context"
+	"fmt"
+	"github.com/gorilla/mux"
+	prometheus2 "github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	metric2 "go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/sdk/metric"
+	"net"
+	"net/http"
+	"reflect"
+)
+
+const defaultEndpoint = "/metrics"
+
+// AppMetrics is metrics interface
+type AppMetrics interface {
+	GetMeter() metric2.Meter
+	Close() error
+	Expose(port int, endpoint string) error
+}
+
+// defaultAppMetrics are core application metrics based on OpenTelemetry https://opentelemetry.io/
+type defaultAppMetrics struct {
+	// Meter can be used by different application parts to create counters and measure things
+	Meter    metric2.Meter
+	listener net.Listener
+	ctx      context.Context
+}
+
+// Close stop application metrics HTTP handler and closes listener.
+func (appMetrics *defaultAppMetrics) Close() error {
+	if appMetrics.listener == nil {
+		return nil
+	}
+	return appMetrics.listener.Close()
+}
+
+// Expose metrics on a given port and endpoint. If endpoint is empty a defaultEndpoint one will be used.
+// Exposes metrics in the Prometheus format https://prometheus.io/
+func (appMetrics *defaultAppMetrics) Expose(port int, endpoint string) error {
+	if endpoint == "" {
+		endpoint = defaultEndpoint
+	}
+	rootRouter := mux.NewRouter()
+	rootRouter.Handle(endpoint, promhttp.HandlerFor(
+		prometheus2.DefaultGatherer,
+		promhttp.HandlerOpts{EnableOpenMetrics: true}))
+	listener, err := net.Listen("tcp4", fmt.Sprintf(":%d", port))
+	if err != nil {
+		return err
+	}
+	appMetrics.listener = listener
+	go func() {
+		err := http.Serve(listener, rootRouter)
+		if err != nil {
+			return
+		}
+	}()
+
+	log.Infof("enabled application metrics and exposing on http://%s", listener.Addr().String())
+
+	return nil
+}
+
+// GetMeter returns metrics meter that can be used to add various counters
+func (appMetrics *defaultAppMetrics) GetMeter() metric2.Meter {
+	return appMetrics.Meter
+}
+
+// NewDefaultAppMetrics and expose them via defaultEndpoint on a given HTTP port
+func NewDefaultAppMetrics(ctx context.Context) (AppMetrics, error) {
+	exporter, err := prometheus.New()
+	if err != nil {
+		return nil, err
+	}
+
+	provider := metric.NewMeterProvider(metric.WithReader(exporter))
+	pkg := reflect.TypeOf(defaultEndpoint).PkgPath()
+	meter := provider.Meter(pkg)
+
+	return &defaultAppMetrics{Meter: meter, ctx: ctx}, nil
+}

management/server/metrics/middleware.go

@@ -0,0 +1,175 @@
+package metrics
+
+import (
+	"context"
+	"fmt"
+	log "github.com/sirupsen/logrus"
+	"go.opentelemetry.io/otel/metric/instrument"
+	"go.opentelemetry.io/otel/metric/instrument/syncint64"
+	"hash/fnv"
+	"net/http"
+	"strings"
+)
+
+const (
+	httpRequestCounterPrefix  = "management.http.request.counter"
+	httpResponseCounterPrefix = "management.http.response.counter"
+)
+
+// WrappedResponseWriter is a wrapper for http.ResponseWriter that allows the
+// written HTTP status code to be captured for metrics reporting or logging purposes.
+type WrappedResponseWriter struct {
+	http.ResponseWriter
+	status      int
+	wroteHeader bool
+}
+
+// WrapResponseWriter wraps original http.ResponseWriter
+func WrapResponseWriter(w http.ResponseWriter) *WrappedResponseWriter {
+	return &WrappedResponseWriter{ResponseWriter: w}
+}
+
+// Status returns response status
+func (rw *WrappedResponseWriter) Status() int {
+	return rw.status
+}
+
+// WriteHeader wraps http.ResponseWriter.WriteHeader method
+func (rw *WrappedResponseWriter) WriteHeader(code int) {
+	if rw.wroteHeader {
+		return
+	}
+
+	rw.status = code
+	rw.ResponseWriter.WriteHeader(code)
+	rw.wroteHeader = true
+}
+
+// HTTPMiddleware handler used to collect metrics of every request/response coming to the API.
+// Also adds request tracing (logging).
+type HTTPMiddleware struct {
+	appMetrics AppMetrics
+	ctx        context.Context
+	// defaultEndpoint & method
+	httpRequestCounters map[string]syncint64.Counter
+	// defaultEndpoint & method & status code
+	httpResponseCounters map[string]syncint64.Counter
+	// all HTTP requests
+	totalHTTPRequestsCounter syncint64.Counter
+	// all HTTP responses
+	totalHTTPResponseCounter syncint64.Counter
+	// all HTTP responses by status code
+	totalHTTPResponseCodeCounters map[int]syncint64.Counter
+}
+
+// AddHTTPRequestResponseCounter adds a new meter for an HTTP defaultEndpoint and Method (GET, POST, etc)
+// Creates one request counter and multiple response counters (one per http response status code).
+func (m *HTTPMiddleware) AddHTTPRequestResponseCounter(endpoint string, method string) error {
+	meterKey := getRequestCounterKey(endpoint, method)
+	httpReqCounter, err := m.appMetrics.GetMeter().SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
+	if err != nil {
+		return err
+	}
+	m.httpRequestCounters[meterKey] = httpReqCounter
+	respCodes := []int{200, 204, 400, 401, 403, 404, 500, 502, 503}
+	for _, code := range respCodes {
+		meterKey = getResponseCounterKey(endpoint, method, code)
+		httpRespCounter, err := m.appMetrics.GetMeter().SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
+		if err != nil {
+			return err
+		}
+		m.httpResponseCounters[meterKey] = httpRespCounter
+
+		meterKey = fmt.Sprintf("%s_%d_total", httpResponseCounterPrefix, code)
+		totalHTTPResponseCodeCounter, err := m.appMetrics.GetMeter().SyncInt64().Counter(meterKey, instrument.WithUnit("1"))
+		if err != nil {
+			return err
+		}
+		m.totalHTTPResponseCodeCounters[code] = totalHTTPResponseCodeCounter
+	}
+
+	return nil
+}
+
+// NewMetricsMiddleware creates a new HTTPMiddleware
+func NewMetricsMiddleware(ctx context.Context, appMetrics AppMetrics) (*HTTPMiddleware, error) {
+
+	totalHTTPRequestsCounter, err := appMetrics.GetMeter().SyncInt64().Counter(
+		fmt.Sprintf("%s_total", httpRequestCounterPrefix),
+		instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	totalHTTPResponseCounter, err := appMetrics.GetMeter().SyncInt64().Counter(
+		fmt.Sprintf("%s_total", httpResponseCounterPrefix),
+		instrument.WithUnit("1"))
+	if err != nil {
+		return nil, err
+	}
+	return &HTTPMiddleware{
+			ctx:                           ctx,
+			httpRequestCounters:           map[string]syncint64.Counter{},
+			httpResponseCounters:          map[string]syncint64.Counter{},
+			totalHTTPResponseCodeCounters: map[int]syncint64.Counter{},
+			appMetrics:                    appMetrics,
+			totalHTTPRequestsCounter:      totalHTTPRequestsCounter,
+			totalHTTPResponseCounter:      totalHTTPResponseCounter,
+		},
+		nil
+}
+
+func getRequestCounterKey(endpoint, method string) string {
+	return fmt.Sprintf("%s%s_%s", httpRequestCounterPrefix,
+		strings.ReplaceAll(endpoint, "/", "_"), method)
+}
+
+func getResponseCounterKey(endpoint, method string, status int) string {
+	return fmt.Sprintf("%s%s_%s_%d", httpResponseCounterPrefix,
+		strings.ReplaceAll(endpoint, "/", "_"), method, status)
+}
+
+// Handler logs every request and response and adds the, to metrics.
+func (m *HTTPMiddleware) Handler(h http.Handler) http.Handler {
+	fn := func(rw http.ResponseWriter, r *http.Request) {
+		traceID := hash(fmt.Sprintf("%v", r))
+		log.Tracef("HTTP request %v: %v %v", traceID, r.Method, r.URL)
+
+		metricKey := getRequestCounterKey(r.URL.Path, r.Method)
+
+		if c, ok := m.httpRequestCounters[metricKey]; ok {
+			c.Add(m.ctx, 1)
+		}
+		m.totalHTTPRequestsCounter.Add(m.ctx, 1)
+
+		w := WrapResponseWriter(rw)
+
+		h.ServeHTTP(w, r)
+
+		if w.Status() > 399 {
+			log.Errorf("HTTP response %v: %v %v status %v", traceID, r.Method, r.URL, w.Status())
+		} else {
+			log.Tracef("HTTP response %v: %v %v status %v", traceID, r.Method, r.URL, w.Status())
+		}
+
+		metricKey = getResponseCounterKey(r.URL.Path, r.Method, w.Status())
+		if c, ok := m.httpResponseCounters[metricKey]; ok {
+			c.Add(m.ctx, 1)
+		}
+
+		m.totalHTTPResponseCounter.Add(m.ctx, 1)
+		if c, ok := m.totalHTTPResponseCodeCounters[w.Status()]; ok {
+			c.Add(m.ctx, 1)
+		}
+	}
+
+	return http.HandlerFunc(fn)
+}
+
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	_, err := h.Write([]byte(s))
+	if err != nil {
+		panic(err)
+	}
+	return h.Sum32()
+}

management/server/metrics/selfhosted.go

@@ -17,7 +17,7 @@ import (
 const (
 	// PayloadEvent identifies an event type
 	PayloadEvent = "self-hosted stats"
-	// payloadEndpoint metrics endpoint to send anonymous data
+	// payloadEndpoint metrics defaultEndpoint to send anonymous data
 	payloadEndpoint = "https://metrics.netbird.io"
 	// defaultPushInterval default interval to push metrics
 	defaultPushInterval = 24 * time.Hour