Merge branch 'main' into feat/auto-upgrade

management/server/types/account.go

@@ -8,6 +8,7 @@ import (
 	"slices"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/hashicorp/go-multierror"
@@ -87,6 +88,13 @@ type Account struct {
 	NetworkRouters   []*routerTypes.NetworkRouter     `gorm:"foreignKey:AccountID;references:id"`
 	NetworkResources []*resourceTypes.NetworkResource `gorm:"foreignKey:AccountID;references:id"`
 	Onboarding       AccountOnboarding                `gorm:"foreignKey:AccountID;references:id;constraint:OnDelete:CASCADE"`
+
+	NetworkMapCache *NetworkMapBuilder `gorm:"-"`
+	nmapInitOnce    *sync.Once         `gorm:"-"`
+}
+
+func (a *Account) InitOnce() {
+	a.nmapInitOnce = &sync.Once{}
 }
 
 // this class is used by gorm only
@@ -257,7 +265,6 @@ func (a *Account) GetPeerNetworkMap(
 	metrics *telemetry.AccountManagerMetrics,
 ) *NetworkMap {
 	start := time.Now()
-
 	peer := a.Peers[peerID]
 	if peer == nil {
 		return &NetworkMap{
@@ -301,7 +308,7 @@ func (a *Account) GetPeerNetworkMap(
 	if dnsManagementStatus {
 		var zones []nbdns.CustomZone
 		if peersCustomZone.Domain != "" {
-			records := filterZoneRecordsForPeers(peer, peersCustomZone, peersToConnect)
+			records := filterZoneRecordsForPeers(peer, peersCustomZone, peersToConnectIncludingRouters, expiredPeers)
 			zones = append(zones, nbdns.CustomZone{
 				Domain:  peersCustomZone.Domain,
 				Records: records,
@@ -890,6 +897,8 @@ func (a *Account) Copy() *Account {
 		NetworkRouters:         networkRouters,
 		NetworkResources:       networkResources,
 		Onboarding:             a.Onboarding,
+		NetworkMapCache:        a.NetworkMapCache,
+		nmapInitOnce:           a.nmapInitOnce,
 	}
 }
 
@@ -1049,14 +1058,7 @@ func (a *Account) connResourcesGenerator(ctx context.Context, targetPeer *nbpeer
 	rules := make([]*FirewallRule, 0)
 	peers := make([]*nbpeer.Peer, 0)
 
-	all, err := a.GetGroupAll()
-	if err != nil {
-		log.WithContext(ctx).Errorf("failed to get group all: %v", err)
-		all = &Group{}
-	}
-
 	return func(rule *PolicyRule, groupPeers []*nbpeer.Peer, direction int) {
-			isAll := (len(all.Peers) - 1) == len(groupPeers)
 			for _, peer := range groupPeers {
 				if peer == nil {
 					continue
@@ -1075,10 +1077,6 @@ func (a *Account) connResourcesGenerator(ctx context.Context, targetPeer *nbpeer
 					Protocol:  string(rule.Protocol),
 				}
 
-				if isAll {
-					fr.PeerIP = "0.0.0.0"
-				}
-
 				ruleID := rule.ID + fr.PeerIP + strconv.Itoa(direction) +
 					fr.Protocol + fr.Action + strings.Join(rule.Ports, ",")
 				if _, ok := rulesExists[ruleID]; ok {
@@ -1682,7 +1680,7 @@ func peerSupportsPortRanges(peerVer string) bool {
 }
 
 // filterZoneRecordsForPeers filters DNS records to only include peers to connect.
-func filterZoneRecordsForPeers(peer *nbpeer.Peer, customZone nbdns.CustomZone, peersToConnect []*nbpeer.Peer) []nbdns.SimpleRecord {
+func filterZoneRecordsForPeers(peer *nbpeer.Peer, customZone nbdns.CustomZone, peersToConnect, expiredPeers []*nbpeer.Peer) []nbdns.SimpleRecord {
 	filteredRecords := make([]nbdns.SimpleRecord, 0, len(customZone.Records))
 	peerIPs := make(map[string]struct{})
 
@@ -1693,6 +1691,10 @@ func filterZoneRecordsForPeers(peer *nbpeer.Peer, customZone nbdns.CustomZone, p
 		peerIPs[peerToConnect.IP.String()] = struct{}{}
 	}
 
+	for _, expiredPeer := range expiredPeers {
+		peerIPs[expiredPeer.IP.String()] = struct{}{}
+	}
+
 	for _, record := range customZone.Records {
 		if _, exists := peerIPs[record.RData]; exists {
 			filteredRecords = append(filteredRecords, record)

management/server/types/account_test.go

@@ -845,6 +845,7 @@ func Test_FilterZoneRecordsForPeers(t *testing.T) {
 		peer            *nbpeer.Peer
 		customZone      nbdns.CustomZone
 		peersToConnect  []*nbpeer.Peer
+		expiredPeers    []*nbpeer.Peer
 		expectedRecords []nbdns.SimpleRecord
 	}{
 		{
@@ -857,6 +858,7 @@ func Test_FilterZoneRecordsForPeers(t *testing.T) {
 				},
 			},
 			peersToConnect: []*nbpeer.Peer{},
+			expiredPeers:   []*nbpeer.Peer{},
 			peer:           &nbpeer.Peer{ID: "router", IP: net.ParseIP("10.0.0.100")},
 			expectedRecords: []nbdns.SimpleRecord{
 				{Name: "router.netbird.cloud", Type: int(dns.TypeA), Class: nbdns.DefaultClass, TTL: 300, RData: "10.0.0.100"},
@@ -890,7 +892,8 @@ func Test_FilterZoneRecordsForPeers(t *testing.T) {
 				}
 				return peers
 			}(),
-			peer: &nbpeer.Peer{ID: "router", IP: net.ParseIP("10.0.0.100")},
+			expiredPeers: []*nbpeer.Peer{},
+			peer:         &nbpeer.Peer{ID: "router", IP: net.ParseIP("10.0.0.100")},
 			expectedRecords: func() []nbdns.SimpleRecord {
 				var records []nbdns.SimpleRecord
 				for _, i := range []int{1, 5, 10, 25, 50, 75, 100} {
@@ -924,7 +927,8 @@ func Test_FilterZoneRecordsForPeers(t *testing.T) {
 				{ID: "peer1", IP: net.ParseIP("10.0.0.1"), DNSLabel: "peer1", ExtraDNSLabels: []string{"peer1-alt", "peer1-backup"}},
 				{ID: "peer2", IP: net.ParseIP("10.0.0.2"), DNSLabel: "peer2", ExtraDNSLabels: []string{"peer2-service"}},
 			},
-			peer: &nbpeer.Peer{ID: "router", IP: net.ParseIP("10.0.0.100")},
+			expiredPeers: []*nbpeer.Peer{},
+			peer:         &nbpeer.Peer{ID: "router", IP: net.ParseIP("10.0.0.100")},
 			expectedRecords: []nbdns.SimpleRecord{
 				{Name: "peer1.netbird.cloud", Type: int(dns.TypeA), Class: nbdns.DefaultClass, TTL: 300, RData: "10.0.0.1"},
 				{Name: "peer1-alt.netbird.cloud", Type: int(dns.TypeA), Class: nbdns.DefaultClass, TTL: 300, RData: "10.0.0.1"},
@@ -934,11 +938,35 @@ func Test_FilterZoneRecordsForPeers(t *testing.T) {
 				{Name: "router.netbird.cloud", Type: int(dns.TypeA), Class: nbdns.DefaultClass, TTL: 300, RData: "10.0.0.100"},
 			},
 		},
+		{
+			name: "expired peers are included in DNS entries",
+			customZone: nbdns.CustomZone{
+				Domain: "netbird.cloud.",
+				Records: []nbdns.SimpleRecord{
+					{Name: "peer1.netbird.cloud", Type: int(dns.TypeA), Class: nbdns.DefaultClass, TTL: 300, RData: "10.0.0.1"},
+					{Name: "peer2.netbird.cloud", Type: int(dns.TypeA), Class: nbdns.DefaultClass, TTL: 300, RData: "10.0.0.2"},
+					{Name: "expired-peer.netbird.cloud", Type: int(dns.TypeA), Class: nbdns.DefaultClass, TTL: 300, RData: "10.0.0.99"},
+					{Name: "router.netbird.cloud", Type: int(dns.TypeA), Class: nbdns.DefaultClass, TTL: 300, RData: "10.0.0.100"},
+				},
+			},
+			peersToConnect: []*nbpeer.Peer{
+				{ID: "peer1", IP: net.ParseIP("10.0.0.1")},
+			},
+			expiredPeers: []*nbpeer.Peer{
+				{ID: "expired-peer", IP: net.ParseIP("10.0.0.99")},
+			},
+			peer: &nbpeer.Peer{ID: "router", IP: net.ParseIP("10.0.0.100")},
+			expectedRecords: []nbdns.SimpleRecord{
+				{Name: "peer1.netbird.cloud", Type: int(dns.TypeA), Class: nbdns.DefaultClass, TTL: 300, RData: "10.0.0.1"},
+				{Name: "expired-peer.netbird.cloud", Type: int(dns.TypeA), Class: nbdns.DefaultClass, TTL: 300, RData: "10.0.0.99"},
+				{Name: "router.netbird.cloud", Type: int(dns.TypeA), Class: nbdns.DefaultClass, TTL: 300, RData: "10.0.0.100"},
+			},
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			result := filterZoneRecordsForPeers(tt.peer, tt.customZone, tt.peersToConnect)
+			result := filterZoneRecordsForPeers(tt.peer, tt.customZone, tt.peersToConnect, tt.expiredPeers)
 			assert.Equal(t, len(tt.expectedRecords), len(result))
 			assert.ElementsMatch(t, tt.expectedRecords, result)
 		})

management/server/types/holder.go

@@ -0,0 +1,43 @@
+package types
+
+import (
+	"context"
+	"sync"
+)
+
+type Holder struct {
+	mu       sync.RWMutex
+	accounts map[string]*Account
+}
+
+func NewHolder() *Holder {
+	return &Holder{
+		accounts: make(map[string]*Account),
+	}
+}
+
+func (h *Holder) GetAccount(id string) *Account {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+	return h.accounts[id]
+}
+
+func (h *Holder) AddAccount(account *Account) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.accounts[account.Id] = account
+}
+
+func (h *Holder) LoadOrStoreFunc(id string, accGetter func(context.Context, string) (*Account, error)) (*Account, error) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	if acc, ok := h.accounts[id]; ok {
+		return acc, nil
+	}
+	account, err := accGetter(context.Background(), id)
+	if err != nil {
+		return nil, err
+	}
+	h.accounts[id] = account
+	return account, nil
+}

management/server/types/networkmap.go

@@ -0,0 +1,58 @@
+package types
+
+import (
+	"context"
+
+	nbdns "github.com/netbirdio/netbird/dns"
+	nbpeer "github.com/netbirdio/netbird/management/server/peer"
+	"github.com/netbirdio/netbird/management/server/telemetry"
+)
+
+func (a *Account) initNetworkMapBuilder(validatedPeers map[string]struct{}) {
+	if a.NetworkMapCache != nil {
+		return
+	}
+	a.nmapInitOnce.Do(func() {
+		a.NetworkMapCache = NewNetworkMapBuilder(a, validatedPeers)
+	})
+}
+
+func (a *Account) InitNetworkMapBuilderIfNeeded(validatedPeers map[string]struct{}) {
+	a.initNetworkMapBuilder(validatedPeers)
+}
+
+func (a *Account) GetPeerNetworkMapExp(
+	ctx context.Context,
+	peerID string,
+	peersCustomZone nbdns.CustomZone,
+	validatedPeers map[string]struct{},
+	metrics *telemetry.AccountManagerMetrics,
+) *NetworkMap {
+	a.initNetworkMapBuilder(validatedPeers)
+	return a.NetworkMapCache.GetPeerNetworkMap(ctx, peerID, peersCustomZone, validatedPeers, metrics)
+}
+
+func (a *Account) OnPeerAddedUpdNetworkMapCache(peerId string) error {
+	if a.NetworkMapCache == nil {
+		return nil
+	}
+	return a.NetworkMapCache.OnPeerAddedIncremental(peerId)
+}
+
+func (a *Account) OnPeerDeletedUpdNetworkMapCache(peerId string) error {
+	if a.NetworkMapCache == nil {
+		return nil
+	}
+	return a.NetworkMapCache.OnPeerDeleted(peerId)
+}
+
+func (a *Account) UpdatePeerInNetworkMapCache(peer *nbpeer.Peer) {
+	if a.NetworkMapCache == nil {
+		return
+	}
+	a.NetworkMapCache.UpdatePeer(peer)
+}
+
+func (a *Account) RecalculateNetworkMapCache(validatedPeers map[string]struct{}) {
+	a.initNetworkMapBuilder(validatedPeers)
+}