[client] introduce client-side event aggregation (#6627)

* added an implementation of aggregating memory store Signed-off-by: Dmitri <dmitri.external@netbird.io> * initial support for aggregation of events Signed-off-by: Dmitri <dmitri.external@netbird.io> * added tcp-aggregation test Signed-off-by: Dmitri <dmitri.external@netbird.io> * added manager integration test Signed-off-by: Dmitri <dmitri.external@netbird.io> * added tracking of the number of start-, drop, and end-events in an aggregation window Signed-off-by: Dmitri <dmitri.external@netbird.io> * fixes based on sonarcube checks Signed-off-by: Dmitri <dmitri.external@netbird.io> * regenerated proto files Signed-off-by: Dmitri <dmitri.external@netbird.io> * removed inadvertenly added google proto files Signed-off-by: Dmitri <dmitri.external@netbird.io> * pacifying linter Signed-off-by: Dmitri <dmitri.external@netbird.io> * update test to validate event aggregation over tcp, udp, icmp, and icmpv6 Signed-off-by: Dmitri <dmitri.external@netbird.io> * updated event aggregation test Signed-off-by: Dmitri <dmitri.external@netbird.io> * regenerate protobufs with expected versions of protoc and protoc-gen-go Signed-off-by: Dmitri <dmitri.external@netbird.io> * remove protoc/protoc-gen headers from flow_grpc.pb.go Signed-off-by: Dmitri <dmitri.external@netbird.io> * updated openapi spec Signed-off-by: Dmitri <dmitri.external@netbird.io> * updated openapi NetworkTrafficEvent spec, regenerated types Signed-off-by: Dmitri <dmitri.external@netbird.io> * respond to feedback Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * fixed an issue with how we track events that shouldn't be aggregated Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * fixed mapping of events to protobuf Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * icmp code values in aggregated events do not matter Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * regenerate openapi types Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * added a comment re: unbounded unacked events Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * reset aggregated event type to unknown Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * fix event aggregation test Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * used the source port of the earliest event Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * add tracking of window starts and ends Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * updated openapi spec Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * reverted changes to generate.sh Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * cleanup handling of not-aggregated events + test Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * responded to feedback + small fixes Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * small fix in a test Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * another test Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * force setting non-empty rule id on aggregated events Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * fixed a couple of issues flagged by coderabbit Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * fix spelling Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * handle exhausted retry backoffs Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> --------- Signed-off-by: Dmitri <dmitri.external@netbird.io> Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>
2026-08-31 12:01:29 +02:00 · 2026-07-06 16:04:26 +02:00
parent 91acb8147c
commit 47352e6e45
12 changed files with 1081 additions and 149 deletions
--- a/shared/management/http/api/openapi.yml
+++ b/shared/management/http/api/openapi.yml
@@ -2769,6 +2769,28 @@ components:
          type: integer
          description: "Number of packets transmitted."
          example: 5
+        num_of_starts:
+          type: integer
+          description: "Number of start events."
+          example: 3
+        num_of_ends:
+          type: integer
+          description: "Number of end events."
+          example: 4
+        num_of_drops:
+          type: integer
+          description: "Number of drop events."
+          example: 5
+        window_start:
+          type: string
+          format: date-time
+          description: Timestamp of the start of the aggregation window.
+          example: 2025-03-20T16:23:58.125397Z
+        window_end:
+          type: string
+          format: date-time
+          description: Timestamp of the end of the aggregation window.
+          example: 2025-03-20T16:23:58.125397Z
        events:
          type: array
          description: "List of events that are correlated to this flow (e.g., start, end)."
@@ -2790,6 +2812,11 @@ components:
        - rx_packets
        - tx_bytes
        - tx_packets
+        - num_of_starts
+        - num_of_ends
+        - num_of_drops
+        - window_start
+        - window_end
        - events
    NetworkTrafficEventsResponse:
      type: object
--- a/shared/management/http/api/types.gen.go
+++ b/shared/management/http/api/types.gen.go
@@ -3709,9 +3709,18 @@ type NetworkTrafficEvent struct {
 	Events []NetworkTrafficSubEvent `json:"events"`

 	// FlowId FlowID is the ID of the connection flow. Not unique because it can be the same for multiple events (e.g., start and end of the connection).
-	FlowId string               `json:"flow_id"`
-	Icmp   NetworkTrafficICMP   `json:"icmp"`
-	Policy NetworkTrafficPolicy `json:"policy"`
+	FlowId string             `json:"flow_id"`
+	Icmp   NetworkTrafficICMP `json:"icmp"`
+
+	// NumOfDrops Number of drop events.
+	NumOfDrops int `json:"num_of_drops"`
+
+	// NumOfEnds Number of end events.
+	NumOfEnds int `json:"num_of_ends"`
+
+	// NumOfStarts Number of start events.
+	NumOfStarts int                  `json:"num_of_starts"`
+	Policy      NetworkTrafficPolicy `json:"policy"`

 	// Protocol Protocol is the protocol of the traffic (e.g. 1 = ICMP, 6 = TCP, 17 = UDP, etc.).
 	Protocol int `json:"protocol"`
@@ -3732,6 +3741,12 @@ type NetworkTrafficEvent struct {
 	// TxPackets Number of packets transmitted.
 	TxPackets int                `json:"tx_packets"`
 	User      NetworkTrafficUser `json:"user"`
+
+	// WindowEnd Timestamp of the end of the aggregation window.
+	WindowEnd time.Time `json:"window_end"`
+
+	// WindowStart Timestamp of the start of the aggregation window.
+	WindowStart time.Time `json:"window_start"`
 }

 // NetworkTrafficEventsResponse defines model for NetworkTrafficEventsResponse.