fix: various bugs in observability / OTel (#1564)

backend/internal/tracing/telemetry_controller.go

@@ -0,0 +1,82 @@
+package tracing
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+)
+
+// maxTelemetryBodySize caps the size of a browser telemetry payload we're willing to forward.
+const maxTelemetryBodySize = 1 << 20 // 1 MiB
+
+type telemetryController struct {
+	targetURL string
+	client    *http.Client
+}
+
+// NewTelemetryController registers an endpoint that receives OTLP/HTTP JSON trace payloads from the browser SPA and forwards them to the OpenTelemetry collector the backend already exports to.
+// The route is only registered when the backend is configured to export traces via OTLP to a resolved collector endpoint, otherwise it's left unregistered and requests return 404.
+func NewTelemetryController(router gin.IRouter, rateLimit gin.HandlerFunc) {
+	// Only register the route when the browser's traces can actually be forwarded to a collector.
+	if !FrontendTracingEnabled() {
+		return
+	}
+
+	tc := &telemetryController{
+		targetURL: otlpTracesEndpoint(),
+		client:    &http.Client{Timeout: 10 * time.Second},
+	}
+	router.POST("/internal/telemetry/traces", rateLimit, tc.forwardTraces)
+}
+
+// frontendTracingEnabled reports whether the browser SPA should export traces.
+// It's true only when trace export over OTLP is enabled and an OTLP/HTTP collector endpoint is resolvable: browser spans are forwarded to that collector by the telemetry endpoint, so without it there's nowhere to send them.
+// The frontend reads this flag (via the public app config) to decide whether to start tracing at all, instead of POSTing spans to an endpoint that isn't registered.
+func FrontendTracingEnabled() bool {
+	return os.Getenv("OTEL_TRACES_EXPORTER") == "otlp" && otlpTracesEndpoint() != ""
+}
+
+func (tc *telemetryController) forwardTraces(c *gin.Context) {
+	body := http.MaxBytesReader(c.Writer, c.Request.Body, maxTelemetryBodySize)
+
+	req, err := http.NewRequestWithContext(c.Request.Context(), http.MethodPost, tc.targetURL, body)
+	if err != nil {
+		_ = c.Error(fmt.Errorf("failed to build telemetry forward request: %w", err))
+		c.Status(http.StatusBadGateway)
+		return
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := tc.client.Do(req)
+	if err != nil {
+		_ = c.Error(fmt.Errorf("failed to forward telemetry to collector: %w", err))
+		c.Status(http.StatusBadGateway)
+		return
+	}
+	defer resp.Body.Close()
+	_, _ = io.Copy(io.Discard, resp.Body)
+
+	c.Status(resp.StatusCode)
+}
+
+// otlpTracesEndpoint derives the collector's OTLP/HTTP traces URL from the standard OpenTelemetry environment variables.
+// It returns an empty string when no OTLP endpoint is configured, in which case the telemetry route is not registered.
+// The endpoint must speak OTLP/HTTP (typically port 4318).
+func otlpTracesEndpoint() string {
+	v := os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")
+	if v != "" {
+		return v
+	}
+
+	base := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+	if base == "" {
+		return ""
+	}
+
+	return strings.TrimRight(base, "/") + "/v1/traces"
+}

backend/internal/tracing/tracing.go

@@ -0,0 +1,62 @@
+// Package tracing wraps the global OpenTelemetry tracer for Pocket ID
+// It centralizes the instrumentation name, the span helpers, and the attribute keys so spans are named and tagged consistently across packages
+// The tracer is resolved from the global provider, so every span is a no-op until the embedding app or the runtime binary configures OpenTelemetry
+package tracing
+
+import (
+	"context"
+	"errors"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+
+	"github.com/pocket-id/pocket-id/backend/internal/common"
+)
+
+// tracer delegates to the global provider, picking up the real provider once one is installed
+var tracer = otel.Tracer(common.TracerName)
+
+// Start begins a span as a child of the span in ctx, returning the derived context and the span
+// The caller must end the span, normally with End so the status is set from the operation error
+//
+//nolint:spancheck // the span is intentionally returned for the caller to end
+func Start(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
+	return tracer.Start(ctx, name, opts...)
+}
+
+// End finishes the span, recording err and marking the span failed when err is non-nil
+func End(span trace.Span, err error) {
+	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, err.Error())
+	}
+
+	span.End()
+}
+
+// EndExpected finishes the span like End, but treats any of the benign errors as a normal outcome rather than a failure
+// It is used, for example, for operations whose "not found" result is expected
+func EndExpected(span trace.Span, err error, benign ...error) {
+	for _, b := range benign {
+		if errors.Is(err, b) {
+			span.End()
+			return
+		}
+	}
+
+	End(span, err)
+}
+
+// Fail records err on the span in ctx and marks it failed, without ending it
+// It is used where the operation reports failure through a non-error value, such as a structured protocol error
+func Fail(ctx context.Context, msg string) {
+	span := trace.SpanFromContext(ctx)
+	span.SetStatus(codes.Error, msg)
+}
+
+// JobID returns the attribute for a job ID
+func JobID(v string) attribute.KeyValue {
+	return attribute.String("pocketid.job.id", v)
+}