[client,management] add netflow support to client and update management (#3414)

adds NetFlow functionality to track and log network traffic information between peers, with features including:

- Flow logging for TCP, UDP, and ICMP traffic
- Integration with connection tracking system
- Resource ID tracking in NetFlow events
- DNS and exit node collection configuration
- Flow API and Redis cache in management
- Memory-based flow storage implementation
- Kernel conntrack counters and userspace counters
- TCP state machine improvements for more accurate tracking
- Migration from net.IP to netip.Addr in the userspace firewall

management/server/peer.go

@@ -29,36 +29,6 @@ import (
 	"github.com/netbirdio/netbird/management/server/status"
 )
 
-// PeerSync used as a data object between the gRPC API and AccountManager on Sync request.
-type PeerSync struct {
-	// WireGuardPubKey is a peers WireGuard public key
-	WireGuardPubKey string
-	// Meta is the system information passed by peer, must be always present
-	Meta nbpeer.PeerSystemMeta
-	// UpdateAccountPeers indicate updating account peers,
-	// which occurs when the peer's metadata is updated
-	UpdateAccountPeers bool
-}
-
-// PeerLogin used as a data object between the gRPC API and AccountManager on Login request.
-type PeerLogin struct {
-	// WireGuardPubKey is a peers WireGuard public key
-	WireGuardPubKey string
-	// SSHKey is a peer's ssh key. Can be empty (e.g., old version do not provide it, or this feature is disabled)
-	SSHKey string
-	// Meta is the system information passed by peer, must be always present.
-	Meta nbpeer.PeerSystemMeta
-	// UserID indicates that JWT was used to log in, and it was valid. Can be empty when SetupKey is used or auth is not required.
-	UserID string
-	// SetupKey references to a server.SetupKey to log in. Can be empty when UserID is used or auth is not required.
-	SetupKey string
-	// ConnectionIP is the real IP of the peer
-	ConnectionIP net.IP
-
-	// ExtraDNSLabels is a list of extra DNS labels that the peer wants to use
-	ExtraDNSLabels []string
-}
-
 // GetPeers returns a list of peers under the given account filtering out peers that do not belong to a user if
 // the current user is not an admin.
 func (am *DefaultAccountManager) GetPeers(ctx context.Context, accountID, userID, nameFilter, ipFilter string) ([]*nbpeer.Peer, error) {
@@ -709,7 +679,7 @@ func getFreeIP(ctx context.Context, transaction store.Store, accountID string) (
 }
 
 // SyncPeer checks whether peer is eligible for receiving NetworkMap (authenticated) and returns its NetworkMap if eligible
-func (am *DefaultAccountManager) SyncPeer(ctx context.Context, sync PeerSync, accountID string) (*nbpeer.Peer, *types.NetworkMap, []*posture.Checks, error) {
+func (am *DefaultAccountManager) SyncPeer(ctx context.Context, sync types.PeerSync, accountID string) (*nbpeer.Peer, *types.NetworkMap, []*posture.Checks, error) {
 	start := time.Now()
 	defer func() {
 		log.WithContext(ctx).Debugf("SyncPeer: took %v", time.Since(start))
@@ -784,7 +754,7 @@ func (am *DefaultAccountManager) SyncPeer(ctx context.Context, sync PeerSync, ac
 	return am.getValidatedPeerWithMap(ctx, peerNotValid, accountID, peer)
 }
 
-func (am *DefaultAccountManager) handlePeerLoginNotFound(ctx context.Context, login PeerLogin, err error) (*nbpeer.Peer, *types.NetworkMap, []*posture.Checks, error) {
+func (am *DefaultAccountManager) handlePeerLoginNotFound(ctx context.Context, login types.PeerLogin, err error) (*nbpeer.Peer, *types.NetworkMap, []*posture.Checks, error) {
 	if errStatus, ok := status.FromError(err); ok && errStatus.Type() == status.NotFound {
 		// we couldn't find this peer by its public key which can mean that peer hasn't been registered yet.
 		// Try registering it.
@@ -804,7 +774,7 @@ func (am *DefaultAccountManager) handlePeerLoginNotFound(ctx context.Context, lo
 
 // LoginPeer logs in or registers a peer.
 // If peer doesn't exist the function checks whether a setup key or a user is present and registers a new peer if so.
-func (am *DefaultAccountManager) LoginPeer(ctx context.Context, login PeerLogin) (*nbpeer.Peer, *types.NetworkMap, []*posture.Checks, error) {
+func (am *DefaultAccountManager) LoginPeer(ctx context.Context, login types.PeerLogin) (*nbpeer.Peer, *types.NetworkMap, []*posture.Checks, error) {
 	accountID, err := am.Store.GetAccountIDByPeerPubKey(ctx, login.WireGuardPubKey)
 	if err != nil {
 		return am.handlePeerLoginNotFound(ctx, login, err)
@@ -994,7 +964,7 @@ func processPeerPostureChecks(ctx context.Context, transaction store.Store, poli
 // The NetBird client doesn't have a way to check if the peer needs login besides sending a login request
 // with no JWT token and usually no setup-key. As the client can send up to two login request to check if it is expired
 // and before starting the engine, we do the checks without an account lock to avoid piling up requests.
-func (am *DefaultAccountManager) checkIFPeerNeedsLoginWithoutLock(ctx context.Context, accountID string, login PeerLogin) error {
+func (am *DefaultAccountManager) checkIFPeerNeedsLoginWithoutLock(ctx context.Context, accountID string, login types.PeerLogin) error {
 	peer, err := am.Store.GetPeerByPeerPubKey(ctx, store.LockingStrengthShare, login.WireGuardPubKey)
 	if err != nil {
 		return err
@@ -1237,7 +1207,13 @@ func (am *DefaultAccountManager) UpdateAccountPeers(ctx context.Context, account
 				remotePeerNetworkMap.Merge(proxyNetworkMap)
 			}
 
-			update := toSyncResponse(ctx, nil, p, nil, nil, remotePeerNetworkMap, am.GetDNSDomain(), postureChecks, dnsCache, account.Settings.RoutingPeerDNSResolutionEnabled)
+			extraSetting, err := am.settingsManager.GetExtraSettings(ctx, accountID)
+			if err != nil {
+				log.WithContext(ctx).Errorf("failed to get flow enabled status: %v", err)
+				return
+			}
+
+			update := toSyncResponse(ctx, &Config{}, p, nil, nil, remotePeerNetworkMap, am.GetDNSDomain(), postureChecks, dnsCache, account.Settings.RoutingPeerDNSResolutionEnabled, extraSetting)
 			am.peersUpdateManager.SendUpdate(ctx, p.ID, &UpdateMessage{Update: update, NetworkMap: remotePeerNetworkMap})
 		}(peer)
 	}
@@ -1300,7 +1276,13 @@ func (am *DefaultAccountManager) UpdateAccountPeer(ctx context.Context, accountI
 		remotePeerNetworkMap.Merge(proxyNetworkMap)
 	}
 
-	update := toSyncResponse(ctx, nil, peer, nil, nil, remotePeerNetworkMap, am.GetDNSDomain(), postureChecks, dnsCache, account.Settings.RoutingPeerDNSResolutionEnabled)
+	extraSettings, err := am.settingsManager.GetExtraSettings(ctx, peer.AccountID)
+	if err != nil {
+		log.WithContext(ctx).Errorf("failed to get extra settings: %v", err)
+		return
+	}
+
+	update := toSyncResponse(ctx, &Config{}, peer, nil, nil, remotePeerNetworkMap, am.GetDNSDomain(), postureChecks, dnsCache, account.Settings.RoutingPeerDNSResolutionEnabled, extraSettings)
 	am.peersUpdateManager.SendUpdate(ctx, peer.ID, &UpdateMessage{Update: update, NetworkMap: remotePeerNetworkMap})
 }