Introduce network serial caching in sync fast path, optimize DB reads, and add granular cache invalidation

management/server/telemetry/grpc_metrics.go

@@ -16,6 +16,7 @@ type GRPCMetrics struct {
 	meter                            metric.Meter
 	syncRequestsCounter              metric.Int64Counter
 	syncRequestsBlockedCounter       metric.Int64Counter
+	syncPathCounter                  metric.Int64Counter
 	loginRequestsCounter             metric.Int64Counter
 	loginRequestsBlockedCounter      metric.Int64Counter
 	loginRequestHighLatencyCounter   metric.Int64Counter
@@ -51,6 +52,14 @@ func NewGRPCMetrics(ctx context.Context, meter metric.Meter) (*GRPCMetrics, erro
 		return nil, err
 	}
 
+	syncPathCounter, err := meter.Int64Counter("management.grpc.sync.path.counter",
+		metric.WithUnit("1"),
+		metric.WithDescription("Number of sync requests by the path taken (fast vs slow). Slow-path rows carry a reason label for fast-path misses."),
+	)
+	if err != nil {
+		return nil, err
+	}
+
 	loginRequestsCounter, err := meter.Int64Counter("management.grpc.login.request.counter",
 		metric.WithUnit("1"),
 		metric.WithDescription("Number of login gRPC requests from the peers to authenticate and receive initial configuration and relay credentials"),
@@ -142,6 +151,7 @@ func NewGRPCMetrics(ctx context.Context, meter metric.Meter) (*GRPCMetrics, erro
 		meter:                            meter,
 		syncRequestsCounter:              syncRequestsCounter,
 		syncRequestsBlockedCounter:       syncRequestsBlockedCounter,
+		syncPathCounter:                  syncPathCounter,
 		loginRequestsCounter:             loginRequestsCounter,
 		loginRequestsBlockedCounter:      loginRequestsBlockedCounter,
 		loginRequestHighLatencyCounter:   loginRequestHighLatencyCounter,
@@ -173,6 +183,26 @@ func (grpcMetrics *GRPCMetrics) CountSyncRequestBlocked() {
 	grpcMetrics.syncRequestsBlockedCounter.Add(grpcMetrics.ctx, 1)
 }
 
+// CountFastPathSync increments the sync-path counter for a Sync that took the
+// fast path. Used together with CountSlowPathSync to graph the fast-path hit
+// rate and, via the reason label on the slow-path counts, see where the
+// misses go (android / cache_miss / serial_mismatch / meta_mismatch / ...).
+func (grpcMetrics *GRPCMetrics) CountFastPathSync() {
+	grpcMetrics.syncPathCounter.Add(grpcMetrics.ctx, 1, metric.WithAttributes(attribute.String("path", "fast")))
+}
+
+// CountSlowPathSync increments the sync-path counter for a Sync that fell
+// through to the slow path. reason is a short tag describing why the fast
+// path was skipped; pass "" if the reason is unknown or the Sync never had
+// a chance to attempt the fast path.
+func (grpcMetrics *GRPCMetrics) CountSlowPathSync(reason string) {
+	attrs := []attribute.KeyValue{attribute.String("path", "slow")}
+	if reason != "" {
+		attrs = append(attrs, attribute.String("reason", reason))
+	}
+	grpcMetrics.syncPathCounter.Add(grpcMetrics.ctx, 1, metric.WithAttributes(attrs...))
+}
+
 // CountGetKeyRequest counts the number of gRPC get server key requests coming to the gRPC API
 func (grpcMetrics *GRPCMetrics) CountGetKeyRequest() {
 	grpcMetrics.getKeyRequestsCounter.Add(grpcMetrics.ctx, 1)