routes compacted

management/server/types/network.go

@@ -2,6 +2,7 @@ package types
 
 import (
 	"encoding/binary"
+	"fmt"
 	"math/rand"
 	"net"
 	"sync"
@@ -49,6 +50,83 @@ func (nm *NetworkMap) Merge(other *NetworkMap) {
 	nm.ForwardingRules = util.MergeUnique(nm.ForwardingRules, other.ForwardingRules)
 }
 
+func (nm *NetworkMap) UncompactRoutes() {
+	peers := make(map[string]*nbpeer.Peer, len(nm.Peers)+len(nm.OfflinePeers))
+	for _, p := range nm.Peers {
+		peers[p.ID] = p
+	}
+	uncompactedRoutes := make([]*route.Route, 0)
+	for _, compactRoute := range nm.Routes {
+		if len(compactRoute.ApplicablePeerIDs) == 0 {
+			uncompactedRoutes = append(uncompactedRoutes, compactRoute.Copy())
+			continue
+		}
+
+		for _, peerID := range compactRoute.ApplicablePeerIDs {
+			expandedRoute := compactRoute.Copy()
+			expandedRoute.ID = route.ID(string(compactRoute.ID) + ":" + peerID)
+			peer := peers[peerID]
+			if peer == nil {
+				continue
+			}
+			expandedRoute.Peer = peer.Key
+			expandedRoute.PeerID = peerID
+			uncompactedRoutes = append(uncompactedRoutes, expandedRoute)
+		}
+	}
+
+	nm.Routes = uncompactedRoutes
+}
+
+func (nm *NetworkMap) ValidateApplicablePeerIDs(compactNm *NetworkMap, expectedPermsMap map[string]map[string]bool) error {
+	if compactNm == nil {
+		return fmt.Errorf("compact network map is nil")
+	}
+
+	peerIDSet := make(map[string]struct{})
+	for _, peer := range nm.Peers {
+		peerIDSet[peer.ID] = struct{}{}
+	}
+
+	for _, route := range compactNm.Routes {
+		if len(route.ApplicablePeerIDs) == 0 {
+			continue
+		}
+
+		for _, peerID := range route.ApplicablePeerIDs {
+			if _, exists := peerIDSet[peerID]; !exists {
+				return fmt.Errorf("route %s has applicable peer ID %s that doesn't exist in peer list", route.ID, peerID)
+			}
+		}
+
+		if expectedPermsMap != nil {
+			expected, hasExpected := expectedPermsMap[string(route.ID)]
+			if hasExpected {
+				expectedPeerIDs := make(map[string]struct{})
+				for peerID, shouldAccess := range expected {
+					if shouldAccess {
+						expectedPeerIDs[peerID] = struct{}{}
+					}
+				}
+
+				if len(route.ApplicablePeerIDs) != len(expectedPeerIDs) {
+					return fmt.Errorf("route %s: expected %d applicable peers, got %d",
+						route.ID, len(expectedPeerIDs), len(route.ApplicablePeerIDs))
+				}
+
+				for _, peerID := range route.ApplicablePeerIDs {
+					if _, expected := expectedPeerIDs[peerID]; !expected {
+						return fmt.Errorf("route %s: peer %s should not have access but is in ApplicablePeerIDs",
+							route.ID, peerID)
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
 func mergeUniquePeersByID(peers1, peers2 []*nbpeer.Peer) []*nbpeer.Peer {
 	result := make(map[string]*nbpeer.Peer)
 	for _, peer := range peers1 {

management/server/types/networkmap_golden_test.go

@@ -1067,3 +1067,72 @@ func createTestAccountWithEntities() *types.Account {
 
 	return account
 }
+
+func createAccountFromFile() (*types.Account, error) {
+	accraw := filepath.Join("testdata", "account_cnlf3j3l0ubs738o5d4g.json")
+	data, err := os.ReadFile(accraw)
+	if err != nil {
+		return nil, err
+	}
+	var account types.Account
+	err = json.Unmarshal(data, &account)
+	if err != nil {
+		return nil, err
+	}
+	return &account, nil
+}
+
+func TestGetPeerNetworkMapCompact(t *testing.T) {
+	account, err := createAccountFromFile()
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	validatedPeersMap := make(map[string]struct{}, len(account.Peers))
+	for _, peer := range account.Peers {
+		validatedPeersMap[peer.ID] = struct{}{}
+	}
+	dnsDomain := account.Settings.DNSDomain
+	customZone := account.GetPeersCustomZone(ctx, dnsDomain)
+
+	builder := types.NewNetworkMapBuilder(account, validatedPeersMap)
+
+	testingPeerID := "d3knp53l0ubs738a3n6g"
+
+	regularNm := builder.GetPeerNetworkMap(ctx, testingPeerID, customZone, validatedPeersMap, nil)
+	compactNm := builder.GetPeerNetworkMapCompact(ctx, testingPeerID, customZone, validatedPeersMap, nil)
+
+	compactedJSON, err := json.MarshalIndent(compactNm, "", "  ")
+	require.NoError(t, err)
+
+	compactNm.UncompactRoutes()
+
+	normalizeAndSortNetworkMap(regularNm)
+	normalizeAndSortNetworkMap(compactNm)
+
+	regularJSON, err := json.MarshalIndent(regularNm, "", "  ")
+	require.NoError(t, err)
+
+	regularLn := len(regularJSON)
+	compactLn := len(compactedJSON)
+
+	t.Logf("compacted less on %d percents", 100-int32((float32(compactLn)/float32(regularLn))*100))
+
+	regular := filepath.Join("testdata", "regular_nmap.json")
+
+	err = os.MkdirAll(filepath.Dir(regular), 0755)
+	require.NoError(t, err)
+	err = os.WriteFile(regular, regularJSON, 0644)
+	require.NoError(t, err)
+
+	uncompactedJSON, err := json.MarshalIndent(compactNm, "", "  ")
+	require.NoError(t, err)
+
+	uncompacted := filepath.Join("testdata", "compacted_nmap.json")
+
+	err = os.MkdirAll(filepath.Dir(regular), 0755)
+	require.NoError(t, err)
+	err = os.WriteFile(uncompacted, uncompactedJSON, 0644)
+	require.NoError(t, err)
+
+	require.JSONEq(t, string(regularJSON), string(uncompactedJSON), "regular and uncompacted network maps should be equal")
+}

management/server/types/networkmapbuilder.go

@@ -1045,6 +1045,149 @@ func (b *NetworkMapBuilder) assembleNetworkMap(
 	}
 }
 
+func (b *NetworkMapBuilder) GetPeerNetworkMapCompact(
+	ctx context.Context, peerID string, peersCustomZone nbdns.CustomZone,
+	validatedPeers map[string]struct{}, metrics *telemetry.AccountManagerMetrics,
+) *NetworkMap {
+	start := time.Now()
+	account := b.account.Load()
+
+	peer := account.GetPeer(peerID)
+	if peer == nil {
+		return &NetworkMap{Network: account.Network.Copy()}
+	}
+
+	b.cache.mu.RLock()
+	defer b.cache.mu.RUnlock()
+
+	aclView := b.cache.peerACLs[peerID]
+	routesView := b.cache.peerRoutes[peerID]
+	dnsConfig := b.cache.peerDNS[peerID]
+
+	if aclView == nil || routesView == nil || dnsConfig == nil {
+		return &NetworkMap{Network: account.Network.Copy()}
+	}
+
+	nm := b.assembleNetworkMapCompact(account, peer, aclView, routesView, dnsConfig, peersCustomZone, validatedPeers)
+
+	if metrics != nil {
+		objectCount := int64(len(nm.Peers) + len(nm.OfflinePeers) + len(nm.Routes) + len(nm.FirewallRules) + len(nm.RoutesFirewallRules))
+		metrics.CountNetworkMapObjects(objectCount)
+		metrics.CountGetPeerNetworkMapDuration(time.Since(start))
+
+		if objectCount > 5000 {
+			log.WithContext(ctx).Tracef("account: %s has a total resource count of %d objects from cache",
+				account.Id, objectCount)
+		}
+	}
+
+	return nm
+}
+
+func (b *NetworkMapBuilder) assembleNetworkMapCompact(
+	account *Account, peer *nbpeer.Peer, aclView *PeerACLView, routesView *PeerRoutesView,
+	dnsConfig *nbdns.Config, customZone nbdns.CustomZone, validatedPeers map[string]struct{},
+) *NetworkMap {
+
+	var peersToConnect []*nbpeer.Peer
+	var expiredPeers []*nbpeer.Peer
+
+	for _, peerID := range aclView.ConnectedPeerIDs {
+		if _, ok := validatedPeers[peerID]; !ok {
+			continue
+		}
+
+		peer := b.cache.globalPeers[peerID]
+		if peer == nil {
+			continue
+		}
+
+		expired, _ := peer.LoginExpired(account.Settings.PeerLoginExpiration)
+		if account.Settings.PeerLoginExpirationEnabled && expired {
+			expiredPeers = append(expiredPeers, peer)
+		} else {
+			peersToConnect = append(peersToConnect, peer)
+		}
+	}
+
+	var routes []*route.Route
+	for _, routeID := range routesView.OwnRouteIDs {
+		if route := b.cache.globalRoutes[routeID]; route != nil {
+			routes = append(routes, route)
+		}
+	}
+	otherRouteIDs := slices.Concat(routesView.NetworkResourceIDs, routesView.InheritedRouteIDs)
+
+	type crt struct {
+		route   *route.Route
+		peerIds []string
+	}
+	rtfilter := make(map[string]crt)
+	peerId := peer.ID
+	for _, routeID := range otherRouteIDs {
+		if route := b.cache.globalRoutes[routeID]; route != nil {
+			rid, pid := splitRouteAndPeer(route)
+			if pid == peerId || len(pid) == 0 {
+				routes = append(routes, route)
+				continue
+			}
+			crt := rtfilter[rid]
+			crt.peerIds = append(crt.peerIds, pid)
+			crt.route = route.CopyClean()
+			rtfilter[rid] = crt
+		}
+	}
+
+	for rid, crt := range rtfilter {
+		crt.route.ApplicablePeerIDs = crt.peerIds
+		crt.route.ID = route.ID(rid)
+		routes = append(routes, crt.route)
+	}
+
+	var firewallRules []*FirewallRule
+	for _, ruleID := range aclView.FirewallRuleIDs {
+		if rule := b.cache.globalRules[ruleID]; rule != nil {
+			firewallRules = append(firewallRules, rule)
+		}
+	}
+
+	var routesFirewallRules []*RouteFirewallRule
+	for _, ruleID := range routesView.RouteFirewallRuleIDs {
+		if rule := b.cache.globalRouteRules[ruleID]; rule != nil {
+			routesFirewallRules = append(routesFirewallRules, rule)
+		}
+	}
+
+	finalDNSConfig := *dnsConfig
+	if finalDNSConfig.ServiceEnable && customZone.Domain != "" {
+		var zones []nbdns.CustomZone
+		records := filterZoneRecordsForPeers(peer, customZone, peersToConnect, expiredPeers)
+		zones = append(zones, nbdns.CustomZone{
+			Domain:  customZone.Domain,
+			Records: records,
+		})
+		finalDNSConfig.CustomZones = zones
+	}
+
+	return &NetworkMap{
+		Peers:               peersToConnect,
+		Network:             account.Network.Copy(),
+		Routes:              routes,
+		DNSConfig:           finalDNSConfig,
+		OfflinePeers:        expiredPeers,
+		FirewallRules:       firewallRules,
+		RoutesFirewallRules: routesFirewallRules,
+	}
+}
+
+func splitRouteAndPeer(r *route.Route) (string, string) {
+	parts := strings.Split(string(r.ID), ":")
+	if len(parts) < 2 {
+		return string(r.ID), ""
+	}
+	return parts[0], parts[1]
+}
+
 func (b *NetworkMapBuilder) generateFirewallRuleID(rule *FirewallRule) string {
 	var s strings.Builder
 	s.WriteString(fw)

route/route.go

@@ -109,6 +109,9 @@ type Route struct {
 	AccessControlGroups []string `gorm:"serializer:json"`
 	// SkipAutoApply indicates if this exit node route (0.0.0.0/0) should skip auto-application for client routing
 	SkipAutoApply bool
+	// ApplicablePeerIDs is used in compact network maps to indicate which peers this route applies to
+	// When populated, client should use these IDs to reference peers from the Peers array instead of using Peer/PeerID/Groups
+	ApplicablePeerIDs []string `gorm:"-"`
 }
 
 // EventMeta returns activity event meta related to the route
@@ -144,6 +147,30 @@ func (r *Route) Copy() *Route {
 	return route
 }
 
+// CopyClean copies a route object without the peer-specific part of the ID
+// and peer data
+func (r *Route) CopyClean() *Route {
+	cleanId := strings.Split(string(r.ID), ":")[0]
+	route := &Route{
+		ID:                  ID(cleanId),
+		AccountID:           r.AccountID,
+		Description:         r.Description,
+		NetID:               r.NetID,
+		Network:             r.Network,
+		Domains:             slices.Clone(r.Domains),
+		KeepRoute:           r.KeepRoute,
+		NetworkType:         r.NetworkType,
+		PeerGroups:          slices.Clone(r.PeerGroups),
+		Metric:              r.Metric,
+		Masquerade:          r.Masquerade,
+		Enabled:             r.Enabled,
+		Groups:              slices.Clone(r.Groups),
+		AccessControlGroups: slices.Clone(r.AccessControlGroups),
+		SkipAutoApply:       r.SkipAutoApply,
+	}
+	return route
+}
+
 // Equal compares one route with the other
 func (r *Route) Equal(other *Route) bool {
 	if r == nil && other == nil {