Files
netbird/client/internal/netflow/logger/logger.go
dmitri-netbird 47352e6e45 [client] introduce client-side event aggregation (#6627)
* added an implementation of aggregating memory store

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* initial support for aggregation of events

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* added tcp-aggregation test

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* added manager integration test

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* added tracking of the number of start-, drop, and end-events in an aggregation window

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* fixes based on sonarcube checks

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* regenerated proto files

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* removed inadvertenly added google proto files

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* pacifying linter

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* update test to validate event aggregation over tcp, udp, icmp, and icmpv6

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* updated event aggregation test

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* regenerate protobufs with expected versions of protoc and protoc-gen-go

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* remove protoc/protoc-gen headers from flow_grpc.pb.go

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* updated openapi spec

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* updated openapi NetworkTrafficEvent spec, regenerated types

Signed-off-by: Dmitri <dmitri.external@netbird.io>

* respond to feedback

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* fixed an issue with how we track events that shouldn't be aggregated

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* fixed mapping of events to protobuf

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* icmp code values in aggregated events do not matter

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* regenerate openapi types

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* added a comment re: unbounded unacked events

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* reset aggregated event type to unknown

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* fix event aggregation test

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* used the source port of the earliest event

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* add tracking of window starts and ends

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* updated openapi spec

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* reverted changes to generate.sh

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* cleanup handling of not-aggregated events + test

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* responded to feedback + small fixes

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* small fix in a test

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* another test

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* force setting non-empty rule id on aggregated events

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* fixed a couple of issues flagged by coderabbit

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* fix spelling

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

* handle exhausted retry backoffs

Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>

---------

Signed-off-by: Dmitri <dmitri.external@netbird.io>
Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>
2026-07-06 16:04:26 +02:00

163 lines
3.4 KiB
Go

package logger
import (
"context"
"net/netip"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/internal/netflow/store"
"github.com/netbirdio/netbird/client/internal/netflow/types"
"github.com/netbirdio/netbird/client/internal/peer"
"github.com/netbirdio/netbird/dns"
)
type rcvChan chan *types.EventFields
type Logger struct {
mux sync.Mutex
enabled atomic.Bool
rcvChan atomic.Pointer[rcvChan]
cancel context.CancelFunc
statusRecorder *peer.Status
wgIfaceNet netip.Prefix
wgIfaceNetV6 netip.Prefix
dnsCollection atomic.Bool
exitNodeCollection atomic.Bool
Store types.AggregatingStore
}
func New(statusRecorder *peer.Status, wgIfaceIPNet, wgIfaceIPNetV6 netip.Prefix) *Logger {
return &Logger{
statusRecorder: statusRecorder,
wgIfaceNet: wgIfaceIPNet,
wgIfaceNetV6: wgIfaceIPNetV6,
Store: store.NewAggregatingMemoryStore(),
}
}
func (l *Logger) StoreEvent(flowEvent types.EventFields) {
if !l.enabled.Load() {
return
}
c := l.rcvChan.Load()
if c == nil {
return
}
select {
case *c <- &flowEvent:
default:
// todo: we should collect or log on this
}
}
func (l *Logger) Enable() {
go l.startReceiver()
}
func (l *Logger) startReceiver() {
if l.enabled.Load() {
return
}
l.mux.Lock()
ctx, cancel := context.WithCancel(context.Background())
l.cancel = cancel
l.mux.Unlock()
c := make(rcvChan, 100)
l.rcvChan.Store(&c)
l.enabled.Store(true)
for {
select {
case <-ctx.Done():
log.Info("flow Memory store receiver stopped")
return
case eventFields := <-c:
id := uuid.New()
event := types.Event{
ID: id,
EventFields: *eventFields,
Timestamp: time.Now().UTC(),
}
var isSrcExitNode bool
var isDestExitNode bool
if !l.isOverlayIP(event.SourceIP) {
event.SourceResourceID, isSrcExitNode = l.statusRecorder.CheckRoutes(event.SourceIP)
}
if !l.isOverlayIP(event.DestIP) {
event.DestResourceID, isDestExitNode = l.statusRecorder.CheckRoutes(event.DestIP)
}
if l.shouldStore(eventFields, isSrcExitNode || isDestExitNode) {
l.Store.StoreEvent(&event)
}
}
}
}
func (l *Logger) Close() {
l.stop()
l.Store.Close()
}
func (l *Logger) stop() {
if !l.enabled.Load() {
return
}
l.enabled.Store(false)
l.mux.Lock()
if l.cancel != nil {
l.cancel()
l.cancel = nil
}
l.rcvChan.Store(nil)
l.mux.Unlock()
}
func (l *Logger) ResetAggregationWindow() types.FlowEventAggregator {
return l.Store.ResetAggregationWindow()
}
func (l *Logger) GetEvents() []*types.Event {
return l.Store.GetEvents()
}
func (l *Logger) DeleteEvents(ids []uuid.UUID) {
l.Store.DeleteEvents(ids)
}
func (l *Logger) UpdateConfig(dnsCollection, exitNodeCollection bool) {
l.dnsCollection.Store(dnsCollection)
l.exitNodeCollection.Store(exitNodeCollection)
}
func (l *Logger) isOverlayIP(ip netip.Addr) bool {
return l.wgIfaceNet.Contains(ip) || (l.wgIfaceNetV6.IsValid() && l.wgIfaceNetV6.Contains(ip))
}
func (l *Logger) shouldStore(event *types.EventFields, isExitNode bool) bool {
// check dns collection
if !l.dnsCollection.Load() && event.Protocol == types.UDP &&
(event.DestPort == 53 || event.DestPort == dns.ForwarderClientPort || event.DestPort == dns.ForwarderServerPort) {
return false
}
// check exit node collection
if !l.exitNodeCollection.Load() && isExitNode {
return false
}
return true
}