mirror of
https://github.com/netbirdio/netbird.git
synced 2026-07-16 11:39:57 +00:00
* added an implementation of aggregating memory store Signed-off-by: Dmitri <dmitri.external@netbird.io> * initial support for aggregation of events Signed-off-by: Dmitri <dmitri.external@netbird.io> * added tcp-aggregation test Signed-off-by: Dmitri <dmitri.external@netbird.io> * added manager integration test Signed-off-by: Dmitri <dmitri.external@netbird.io> * added tracking of the number of start-, drop, and end-events in an aggregation window Signed-off-by: Dmitri <dmitri.external@netbird.io> * fixes based on sonarcube checks Signed-off-by: Dmitri <dmitri.external@netbird.io> * regenerated proto files Signed-off-by: Dmitri <dmitri.external@netbird.io> * removed inadvertenly added google proto files Signed-off-by: Dmitri <dmitri.external@netbird.io> * pacifying linter Signed-off-by: Dmitri <dmitri.external@netbird.io> * update test to validate event aggregation over tcp, udp, icmp, and icmpv6 Signed-off-by: Dmitri <dmitri.external@netbird.io> * updated event aggregation test Signed-off-by: Dmitri <dmitri.external@netbird.io> * regenerate protobufs with expected versions of protoc and protoc-gen-go Signed-off-by: Dmitri <dmitri.external@netbird.io> * remove protoc/protoc-gen headers from flow_grpc.pb.go Signed-off-by: Dmitri <dmitri.external@netbird.io> * updated openapi spec Signed-off-by: Dmitri <dmitri.external@netbird.io> * updated openapi NetworkTrafficEvent spec, regenerated types Signed-off-by: Dmitri <dmitri.external@netbird.io> * respond to feedback Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * fixed an issue with how we track events that shouldn't be aggregated Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * fixed mapping of events to protobuf Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * icmp code values in aggregated events do not matter Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * regenerate openapi types Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * added a comment re: unbounded unacked events Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * reset aggregated event type to unknown Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * fix event aggregation test Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * used the source port of the earliest event Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * add tracking of window starts and ends Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * updated openapi spec Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * reverted changes to generate.sh Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * cleanup handling of not-aggregated events + test Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * responded to feedback + small fixes Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * small fix in a test Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * another test Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * force setting non-empty rule id on aggregated events Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * fixed a couple of issues flagged by coderabbit Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * fix spelling Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> * handle exhausted retry backoffs Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io> --------- Signed-off-by: Dmitri <dmitri.external@netbird.io> Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>
163 lines
3.4 KiB
Go
163 lines
3.4 KiB
Go
package logger
|
|
|
|
import (
|
|
"context"
|
|
"net/netip"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
log "github.com/sirupsen/logrus"
|
|
|
|
"github.com/netbirdio/netbird/client/internal/netflow/store"
|
|
"github.com/netbirdio/netbird/client/internal/netflow/types"
|
|
"github.com/netbirdio/netbird/client/internal/peer"
|
|
"github.com/netbirdio/netbird/dns"
|
|
)
|
|
|
|
type rcvChan chan *types.EventFields
|
|
type Logger struct {
|
|
mux sync.Mutex
|
|
enabled atomic.Bool
|
|
rcvChan atomic.Pointer[rcvChan]
|
|
cancel context.CancelFunc
|
|
statusRecorder *peer.Status
|
|
wgIfaceNet netip.Prefix
|
|
wgIfaceNetV6 netip.Prefix
|
|
dnsCollection atomic.Bool
|
|
exitNodeCollection atomic.Bool
|
|
Store types.AggregatingStore
|
|
}
|
|
|
|
func New(statusRecorder *peer.Status, wgIfaceIPNet, wgIfaceIPNetV6 netip.Prefix) *Logger {
|
|
return &Logger{
|
|
statusRecorder: statusRecorder,
|
|
wgIfaceNet: wgIfaceIPNet,
|
|
wgIfaceNetV6: wgIfaceIPNetV6,
|
|
Store: store.NewAggregatingMemoryStore(),
|
|
}
|
|
}
|
|
|
|
func (l *Logger) StoreEvent(flowEvent types.EventFields) {
|
|
if !l.enabled.Load() {
|
|
return
|
|
}
|
|
|
|
c := l.rcvChan.Load()
|
|
if c == nil {
|
|
return
|
|
}
|
|
|
|
select {
|
|
case *c <- &flowEvent:
|
|
default:
|
|
// todo: we should collect or log on this
|
|
}
|
|
}
|
|
|
|
func (l *Logger) Enable() {
|
|
go l.startReceiver()
|
|
}
|
|
|
|
func (l *Logger) startReceiver() {
|
|
if l.enabled.Load() {
|
|
return
|
|
}
|
|
|
|
l.mux.Lock()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
l.cancel = cancel
|
|
l.mux.Unlock()
|
|
|
|
c := make(rcvChan, 100)
|
|
l.rcvChan.Store(&c)
|
|
l.enabled.Store(true)
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
log.Info("flow Memory store receiver stopped")
|
|
return
|
|
case eventFields := <-c:
|
|
id := uuid.New()
|
|
event := types.Event{
|
|
ID: id,
|
|
EventFields: *eventFields,
|
|
Timestamp: time.Now().UTC(),
|
|
}
|
|
|
|
var isSrcExitNode bool
|
|
var isDestExitNode bool
|
|
|
|
if !l.isOverlayIP(event.SourceIP) {
|
|
event.SourceResourceID, isSrcExitNode = l.statusRecorder.CheckRoutes(event.SourceIP)
|
|
}
|
|
|
|
if !l.isOverlayIP(event.DestIP) {
|
|
event.DestResourceID, isDestExitNode = l.statusRecorder.CheckRoutes(event.DestIP)
|
|
}
|
|
|
|
if l.shouldStore(eventFields, isSrcExitNode || isDestExitNode) {
|
|
l.Store.StoreEvent(&event)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (l *Logger) Close() {
|
|
l.stop()
|
|
l.Store.Close()
|
|
}
|
|
|
|
func (l *Logger) stop() {
|
|
if !l.enabled.Load() {
|
|
return
|
|
}
|
|
|
|
l.enabled.Store(false)
|
|
l.mux.Lock()
|
|
if l.cancel != nil {
|
|
l.cancel()
|
|
l.cancel = nil
|
|
}
|
|
l.rcvChan.Store(nil)
|
|
l.mux.Unlock()
|
|
}
|
|
|
|
func (l *Logger) ResetAggregationWindow() types.FlowEventAggregator {
|
|
return l.Store.ResetAggregationWindow()
|
|
}
|
|
|
|
func (l *Logger) GetEvents() []*types.Event {
|
|
return l.Store.GetEvents()
|
|
}
|
|
|
|
func (l *Logger) DeleteEvents(ids []uuid.UUID) {
|
|
l.Store.DeleteEvents(ids)
|
|
}
|
|
|
|
func (l *Logger) UpdateConfig(dnsCollection, exitNodeCollection bool) {
|
|
l.dnsCollection.Store(dnsCollection)
|
|
l.exitNodeCollection.Store(exitNodeCollection)
|
|
}
|
|
|
|
func (l *Logger) isOverlayIP(ip netip.Addr) bool {
|
|
return l.wgIfaceNet.Contains(ip) || (l.wgIfaceNetV6.IsValid() && l.wgIfaceNetV6.Contains(ip))
|
|
}
|
|
|
|
func (l *Logger) shouldStore(event *types.EventFields, isExitNode bool) bool {
|
|
// check dns collection
|
|
if !l.dnsCollection.Load() && event.Protocol == types.UDP &&
|
|
(event.DestPort == 53 || event.DestPort == dns.ForwarderClientPort || event.DestPort == dns.ForwarderServerPort) {
|
|
return false
|
|
}
|
|
|
|
// check exit node collection
|
|
if !l.exitNodeCollection.Load() && isExitNode {
|
|
return false
|
|
}
|
|
|
|
return true
|
|
}
|