sendnrw/netbird
2
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2026-10-04 20:49:06 +02:00
Code Issues Packages Projects Releases Wiki Activity

replaced sequential IDs with xids (random, uuid-like)

Browse Source 
Signed-off-by: Dmitri Dolguikh <dmitri.external@netbird.io>
This commit is contained in:
Dmitri Dolguikh
2026-07-09 16:15:58 +02:00
parent 5edc168ff1
commit 7cae1b9dd1
36 changed files with 378 additions and 1554 deletions
Download Patch File Download Diff File Expand all files Collapse all files
management/internals/shared/grpc/components_encoder.go
+69 -86
View File
@@ -199,9 +199,6 @@ func (e *componentEncoder) encodeGroups() []*proto.GroupCompact {
out := make([]*proto.GroupCompact, 0, len(e.components.Groups))
for _, g := range e.components.Groups {
if !g.HasSeqID() {
continue
}
peerIdxs := make([]uint32, 0, len(g.Peers))
for _, peerID := range g.Peers {
if idx, ok := e.peerOrder[peerID]; ok {
@@ -209,7 +206,7 @@ func (e *componentEncoder) encodeGroups() []*proto.GroupCompact {
}
}
out = append(out, &proto.GroupCompact{
Id: g.AccountSeqID,
Id: g.PublicID,
Name: g.Name,
PeerIndexes: peerIdxs,
})
@@ -229,7 +226,7 @@ func (e *componentEncoder) encodePolicies(policies []*types.Policy) []*proto.Pol
out := make([]*proto.PolicyCompact, 0, len(policies))
for _, pol := range policies {
if !pol.HasSeqID() || !pol.Enabled {
if !pol.Enabled {
continue
}
for _, r := range pol.Rules {
@@ -245,14 +242,14 @@ func (e *componentEncoder) encodePolicies(policies []*types.Policy) []*proto.Pol
// encodePolicyRule maps a single PolicyRule under pol to a PolicyCompact entry.
func (e *componentEncoder) encodePolicyRule(pol *types.Policy, r *types.PolicyRule) *proto.PolicyCompact {
return &proto.PolicyCompact{
Id: pol.AccountSeqID,
Id: pol.PublicID,
Action: networkmap.GetProtoAction(string(r.Action)),
Protocol: networkmap.GetProtoProtocol(string(r.Protocol)),
Bidirectional: r.Bidirectional,
Ports: portsToUint32(r.Ports),
PortRanges: portRangesToProto(r.PortRanges),
SourceGroupIds: e.groupSeqIDs(r.Sources),
DestinationGroupIds: e.groupSeqIDs(r.Destinations),
SourceGroupIds: e.groupPublicXids(r.Sources),
DestinationGroupIds: e.groupPublicXids(r.Destinations),
AuthorizedUser: r.AuthorizedUser,
AuthorizedGroups: e.encodeAuthorizedGroups(r.AuthorizedGroups),
SourceResource: e.resourceToProto(r.SourceResource),
@@ -261,16 +258,16 @@ func (e *componentEncoder) encodePolicyRule(pol *types.Policy, r *types.PolicyRu
}
}
// groupSeqIDs maps the xid group IDs in src to their per-account seq ids,
// dropping any group that has no seq id assigned.
func (e *componentEncoder) groupSeqIDs(src []string) []int32 {
// groupPublicXids maps the xid group IDs in src to their public xids,
// dropping any group with invalid public xid.
func (e *componentEncoder) groupPublicXids(src []string) []string {
if len(src) == 0 {
return nil
}
out := make([]int32, 0, len(src))
out := make([]string, 0, len(src))
for _, gid := range src {
if seq, ok := e.groupSeq(gid); ok {
out = append(out, seq)
if id, ok := e.groupPublicXid(gid); ok {
out = append(out, id)
}
}
return out
@@ -319,27 +316,27 @@ func unionPolicies(policies []*types.Policy, resourcePolicies map[string][]*type
// group xid → local-user names) to the wire form (map keyed by group
// account_seq_id → UserNameList). Groups without a seq id are dropped —
// matches how source/destination group references handle the same case.
func (e *componentEncoder) encodeAuthorizedGroups(m map[string][]string) map[int32]*proto.UserNameList {
func (e *componentEncoder) encodeAuthorizedGroups(m map[string][]string) map[string]*proto.UserNameList {
if len(m) == 0 {
return nil
}
out := make(map[int32]*proto.UserNameList, len(m))
out := make(map[string]*proto.UserNameList, len(m))
for groupID, names := range m {
seq, ok := e.groupSeq(groupID)
id, ok := e.groupPublicXid(groupID)
if !ok {
continue
}
out[seq] = &proto.UserNameList{Names: append([]string(nil), names...)}
out[id] = &proto.UserNameList{Names: names}
}
return out
}
func (e *componentEncoder) groupSeq(groupID string) (int32, bool) {
func (e *componentEncoder) groupPublicXid(groupID string) (string, bool) {
g, ok := e.components.Groups[groupID]
if !ok || !g.HasSeqID() {
return 0, false
if !ok {
return "", false
}
return g.AccountSeqID, true
return g.PublicID, true
}
// resourceToProto translates types.Resource for the wire. For peer-typed
@@ -362,34 +359,33 @@ func (e *componentEncoder) resourceToProto(r types.Resource) *proto.ResourceComp
}
// postureCheckSeqs translates a slice of posture-check xids to their
// per-account integer ids using the NetworkMapComponents.PostureCheckXIDToSeq
// lookup. Unresolvable xids are silently dropped — matches how group/peer
// public xids. Unresolvable xids are silently dropped — matches how group/peer
// references handle the same case.
func (e *componentEncoder) postureCheckSeqs(xids []string) []int32 {
if len(xids) == 0 || len(e.components.PostureCheckXIDToSeq) == 0 {
func (e *componentEncoder) postureCheckSeqs(xids []string) []string {
if len(xids) == 0 || len(e.components.PostureCheckXIDToPublicID) == 0 {
return nil
}
out := make([]int32, 0, len(xids))
out := make([]string, 0, len(xids))
for _, xid := range xids {
if seq, ok := e.components.PostureCheckXIDToSeq[xid]; ok {
if seq, ok := e.components.PostureCheckXIDToPublicID[xid]; ok {
out = append(out, seq)
}
}
return out
}
// networkSeq translates a Network xid to its per-account integer id using
// the NetworkMapComponents.NetworkXIDToSeq lookup. Returns (0,false) when
// networkSeq translates a Network xid to its public id using
// the NetworkMapComponents.NetworkXIDToPublicID lookup. Returns (0,false) when
// the xid isn't known — callers decide whether to skip the parent record.
func (e *componentEncoder) networkSeq(xid string) (int32, bool) {
func (e *componentEncoder) networkPublicId(xid string) (string, bool) {
if xid == "" {
return 0, false
return "", false
}
seq, ok := e.components.NetworkXIDToSeq[xid]
if !ok || seq == 0 {
return 0, false
id, ok := e.components.NetworkXIDToPublicID[xid]
if !ok {
return "", false
}
return seq, true
return id, true
}
func (e *componentEncoder) encodeDNSSettings(s *types.DNSSettings) *proto.DNSSettingsCompact {
@@ -397,11 +393,11 @@ func (e *componentEncoder) encodeDNSSettings(s *types.DNSSettings) *proto.DNSSet
return nil
}
out := &proto.DNSSettingsCompact{
DisabledManagementGroupIds: make([]int32, 0, len(s.DisabledManagementGroups)),
DisabledManagementGroupIds: make([]string, 0, len(s.DisabledManagementGroups)),
}
for _, gid := range s.DisabledManagementGroups {
if seq, ok := e.groupSeq(gid); ok {
out.DisabledManagementGroupIds = append(out.DisabledManagementGroupIds, seq)
if id, ok := e.groupPublicXid(gid); ok {
out.DisabledManagementGroupIds = append(out.DisabledManagementGroupIds, id)
}
}
return out
@@ -417,7 +413,7 @@ func (e *componentEncoder) encodeRoutes(routes []*nbroute.Route) []*proto.RouteR
continue
}
rr := &proto.RouteRaw{
Id: r.AccountSeqID,
Id: r.PublicID,
NetId: string(r.NetID),
Description: r.Description,
KeepRoute: r.KeepRoute,
@@ -427,9 +423,9 @@ func (e *componentEncoder) encodeRoutes(routes []*nbroute.Route) []*proto.RouteR
Enabled: r.Enabled,
SkipAutoApply: r.SkipAutoApply,
Domains: r.Domains.ToPunycodeList(),
GroupIds: e.groupIDsToSeq(r.Groups),
AccessControlGroupIds: e.groupIDsToSeq(r.AccessControlGroups),
PeerGroupIds: e.groupIDsToSeq(r.PeerGroups),
GroupIds: e.groupPublicXids(r.Groups),
AccessControlGroupIds: e.groupPublicXids(r.AccessControlGroups),
PeerGroupIds: e.groupPublicXids(r.PeerGroups),
}
if r.Network.IsValid() {
rr.NetworkCidr = r.Network.String()
@@ -445,19 +441,6 @@ func (e *componentEncoder) encodeRoutes(routes []*nbroute.Route) []*proto.RouteR
return out
}
func (e *componentEncoder) groupIDsToSeq(groupIDs []string) []int32 {
if len(groupIDs) == 0 {
return nil
}
out := make([]int32, 0, len(groupIDs))
for _, gid := range groupIDs {
if seq, ok := e.groupSeq(gid); ok {
out = append(out, seq)
}
}
return out
}
func (e *componentEncoder) encodeNameServerGroups(nsgs []*nbdns.NameServerGroup) []*proto.NameServerGroupRaw {
if len(nsgs) == 0 {
return nil
@@ -468,11 +451,11 @@ func (e *componentEncoder) encodeNameServerGroups(nsgs []*nbdns.NameServerGroup)
continue
}
entry := &proto.NameServerGroupRaw{
Id: nsg.AccountSeqID,
Id: nsg.PublicID,
Name: nsg.Name,
Description: nsg.Description,
Nameservers: encodeNameServers(nsg.NameServers),
GroupIds: e.groupIDsToSeq(nsg.Groups),
GroupIds: e.groupPublicXids(nsg.Groups),
Primary: nsg.Primary,
Domains: nsg.Domains,
Enabled: nsg.Enabled,
@@ -541,7 +524,7 @@ func (e *componentEncoder) encodeNetworkResources(resources []*resourceTypes.Net
continue
}
entry := &proto.NetworkResourceRaw{
Id: r.AccountSeqID,
Id: r.PublicID,
Name: r.Name,
Description: r.Description,
Type: string(r.Type),
@@ -549,8 +532,8 @@ func (e *componentEncoder) encodeNetworkResources(resources []*resourceTypes.Net
DomainValue: r.Domain,
Enabled: r.Enabled,
}
if seq, ok := e.networkSeq(r.NetworkID); ok {
entry.NetworkSeq = seq
if id, ok := e.networkPublicId(r.NetworkID); ok {
entry.NetworkSeq = id
}
if r.Prefix.IsValid() {
entry.PrefixCidr = r.Prefix.String()
@@ -560,16 +543,16 @@ func (e *componentEncoder) encodeNetworkResources(resources []*resourceTypes.Net
return out
}
func (e *componentEncoder) encodeRoutersMap(routersMap map[string]map[string]*routerTypes.NetworkRouter) map[int32]*proto.NetworkRouterList {
func (e *componentEncoder) encodeRoutersMap(routersMap map[string]map[string]*routerTypes.NetworkRouter) map[string]*proto.NetworkRouterList {
if len(routersMap) == 0 {
return nil
}
out := make(map[int32]*proto.NetworkRouterList, len(routersMap))
out := make(map[string]*proto.NetworkRouterList, len(routersMap))
for networkXID, routers := range routersMap {
if len(routers) == 0 {
continue
}
netSeq, ok := e.networkSeq(networkXID)
id, ok := e.networkPublicId(networkXID)
if !ok {
continue
}
@@ -579,8 +562,8 @@ func (e *componentEncoder) encodeRoutersMap(routersMap map[string]map[string]*ro
continue
}
entry := &proto.NetworkRouterEntry{
Id: r.AccountSeqID,
PeerGroupIds: e.groupIDsToSeq(r.PeerGroups),
Id: r.PublicID,
PeerGroupIds: e.groupPublicXids(r.PeerGroups),
Masquerade: r.Masquerade,
Metric: int32(r.Metric),
Enabled: r.Enabled,
@@ -591,53 +574,53 @@ func (e *componentEncoder) encodeRoutersMap(routersMap map[string]map[string]*ro
}
entries = append(entries, entry)
}
out[netSeq] = &proto.NetworkRouterList{Entries: entries}
out[id] = &proto.NetworkRouterList{Entries: entries}
}
return out
}
func (e *componentEncoder) encodeResourcePoliciesMap(rpm map[string][]*types.Policy) map[int32]*proto.PolicyIds {
func (e *componentEncoder) encodeResourcePoliciesMap(rpm map[string][]*types.Policy) map[string]*proto.PolicyIds {
if len(rpm) == 0 {
return nil
}
// resourceXIDToSeq is local to one encode — built from components.NetworkResources
// resourceXIDToPublicID is local to one encode — built from components.NetworkResources
// (small slice). Network resources without seq id are dropped, matching how
// other components-without-seq are silently filtered.
resourceXIDToSeq := make(map[string]int32, len(e.components.NetworkResources))
resourceXIDToPublicID := make(map[string]string, len(e.components.NetworkResources))
for _, r := range e.components.NetworkResources {
if r != nil && r.AccountSeqID != 0 {
resourceXIDToSeq[r.ID] = r.AccountSeqID
if r != nil {
resourceXIDToPublicID[r.ID] = r.PublicID
}
}
out := make(map[int32]*proto.PolicyIds, len(rpm))
out := make(map[string]*proto.PolicyIds, len(rpm))
for resourceXID, policies := range rpm {
seq, ok := resourceXIDToSeq[resourceXID]
resId, ok := resourceXIDToPublicID[resourceXID]
if !ok {
continue
}
ids := make([]int32, 0, len(policies))
ids := make([]string, 0, len(policies))
for _, pol := range policies {
ids = append(ids, pol.AccountSeqID)
ids = append(ids, pol.PublicID)
}
if len(ids) == 0 {
continue
}
out[seq] = &proto.PolicyIds{Ids: ids}
out[resId] = &proto.PolicyIds{Ids: ids}
}
return out
}
func (e *componentEncoder) encodeGroupIDToUserIDs(m map[string][]string) map[int32]*proto.UserIDList {
func (e *componentEncoder) encodeGroupIDToUserIDs(m map[string][]string) map[string]*proto.UserIDList {
if len(m) == 0 {
return nil
}
out := make(map[int32]*proto.UserIDList, len(m))
out := make(map[string]*proto.UserIDList, len(m))
for groupID, userIDs := range m {
seq, ok := e.groupSeq(groupID)
id, ok := e.groupPublicXid(groupID)
if !ok || len(userIDs) == 0 {
continue
}
out[seq] = &proto.UserIDList{UserIds: userIDs}
out[id] = &proto.UserIDList{UserIds: userIDs}
}
return out
}
@@ -653,14 +636,14 @@ func stringSetToSlice(s map[string]struct{}) []string {
return out
}
func (e *componentEncoder) encodePostureFailedPeers(m map[string]map[string]struct{}) map[int32]*proto.PeerIndexSet {
func (e *componentEncoder) encodePostureFailedPeers(m map[string]map[string]struct{}) map[string]*proto.PeerIndexSet {
if len(m) == 0 {
return nil
}
out := make(map[int32]*proto.PeerIndexSet, len(m))
out := make(map[string]*proto.PeerIndexSet, len(m))
for checkXID, failedPeerIDs := range m {
seq, ok := e.components.PostureCheckXIDToSeq[checkXID]
if !ok || seq == 0 {
id, ok := e.components.PostureCheckXIDToPublicID[checkXID]
if !ok {
continue
}
idxs := make([]uint32, 0, len(failedPeerIDs))
@@ -672,7 +655,7 @@ func (e *componentEncoder) encodePostureFailedPeers(m map[string]map[string]stru
if len(idxs) == 0 {
continue
}
out[seq] = &proto.PeerIndexSet{PeerIndexes: idxs}
out[id] = &proto.PeerIndexSet{PeerIndexes: idxs}
}
return out
}
management/internals/shared/grpc/components_encoder_test.go
+59 -97
View File
@@ -197,14 +197,14 @@ func newTestComponents() *types.NetworkMapComponents {
"peer-c": peerC,
},
Groups: map[string]*types.Group{
"group-src": {ID: "group-src", AccountSeqID: 1, Name: "Src", Peers: []string{"peer-a"}},
"group-dst": {ID: "group-dst", AccountSeqID: 2, Name: "Dst", Peers: []string{"peer-b", "peer-c"}},
"group-src": {ID: "group-src", PublicID: "1", Name: "Src", Peers: []string{"peer-a"}},
"group-dst": {ID: "group-dst", PublicID: "2", Name: "Dst", Peers: []string{"peer-b", "peer-c"}},
},
Policies: []*types.Policy{
{
ID: "pol-1",
AccountSeqID: 10,
Enabled: true,
ID: "pol-1",
PublicID: "10",
Enabled: true,
Rules: []*types.PolicyRule{{
ID: "rule-1", Enabled: true, Action: types.PolicyTrafficActionAccept,
Protocol: types.PolicyRuleProtocolTCP, Bidirectional: true,
@@ -291,23 +291,23 @@ func TestEncodeNetworkMapEnvelope_ConcurrentEncodesEquivalent(t *testing.T) {
}
}
func TestEncodeNetworkMapEnvelope_GroupsByAccountSeqID(t *testing.T) {
func TestEncodeNetworkMapEnvelope_GroupsByAccountPublicId(t *testing.T) {
c := newTestComponents()
full := EncodeNetworkMapEnvelope(ComponentsEnvelopeInput{Components: c}).GetFull()
require.Len(t, full.Groups, 2)
groupByID := map[int32]*proto.GroupCompact{}
groupByID := map[string]*proto.GroupCompact{}
for _, g := range full.Groups {
groupByID[g.Id] = g
}
require.Contains(t, groupByID, int32(1))
require.Contains(t, groupByID, int32(2))
assert.Equal(t, "Src", groupByID[1].Name)
assert.Equal(t, "Dst", groupByID[2].Name)
assert.Len(t, groupByID[1].PeerIndexes, 1)
assert.Len(t, groupByID[2].PeerIndexes, 2)
require.Contains(t, groupByID, "1")
require.Contains(t, groupByID, "2")
assert.Equal(t, "Src", groupByID["1"].Name)
assert.Equal(t, "Dst", groupByID["2"].Name)
assert.Len(t, groupByID["1"].PeerIndexes, 1)
assert.Len(t, groupByID["2"].PeerIndexes, 2)
}
func TestEncodeNetworkMapEnvelope_PolicyExpansion(t *testing.T) {
@@ -317,7 +317,7 @@ func TestEncodeNetworkMapEnvelope_PolicyExpansion(t *testing.T) {
require.Len(t, full.Policies, 1)
pc := full.Policies[0]
assert.EqualValues(t, 10, pc.Id)
assert.EqualValues(t, "10", pc.Id)
assert.Equal(t, proto.RuleAction_ACCEPT, pc.Action)
assert.Equal(t, proto.RuleProtocol_TCP, pc.Protocol)
assert.True(t, pc.Bidirectional)
@@ -325,8 +325,8 @@ func TestEncodeNetworkMapEnvelope_PolicyExpansion(t *testing.T) {
require.Len(t, pc.PortRanges, 1)
assert.EqualValues(t, 8000, pc.PortRanges[0].Start)
assert.EqualValues(t, 8100, pc.PortRanges[0].End)
assert.Equal(t, []int32{1}, pc.SourceGroupIds)
assert.Equal(t, []int32{2}, pc.DestinationGroupIds)
assert.Equal(t, []string{"1"}, pc.SourceGroupIds)
assert.Equal(t, []string{"2"}, pc.DestinationGroupIds)
}
func TestEncodeNetworkMapEnvelope_RouterIndexes(t *testing.T) {
@@ -349,21 +349,6 @@ func TestEncodeNetworkMapEnvelope_DisabledPolicySkipped(t *testing.T) {
assert.Empty(t, full.Policies)
}
func TestEncodeNetworkMapEnvelope_GroupZeroSeqIDSkipped(t *testing.T) {
c := newTestComponents()
c.Groups["group-src"].AccountSeqID = 0
full := EncodeNetworkMapEnvelope(ComponentsEnvelopeInput{Components: c}).GetFull()
require.Len(t, full.Groups, 1, "groups with AccountSeqID=0 are not yet persisted and must be skipped")
assert.EqualValues(t, 2, full.Groups[0].Id)
require.Len(t, full.Policies, 1)
pc := full.Policies[0]
assert.Empty(t, pc.SourceGroupIds, "rule references a group that was filtered out → no group id on wire")
assert.Equal(t, []int32{2}, pc.DestinationGroupIds)
}
func TestEncodeNetworkMapEnvelope_TwoPeersSameMalformedKey(t *testing.T) {
// Both peers have nil WgPubKey after decode; canonicalize must still
// produce a stable order using DnsLabel as a tiebreaker, so 100 encodes
@@ -496,7 +481,7 @@ func TestEncodeNetworkMapEnvelope_RoutesRoundTrip(t *testing.T) {
c.Routes = []*nbroute.Route{
{
ID: "route-peer",
AccountSeqID: 100,
PublicID: "100",
NetID: "net-A",
Description: "via peer-c",
Network: netip.MustParsePrefix("10.0.0.0/16"),
@@ -506,24 +491,18 @@ func TestEncodeNetworkMapEnvelope_RoutesRoundTrip(t *testing.T) {
Enabled: true,
},
{
ID: "route-peergroup",
AccountSeqID: 101,
NetID: "net-B",
Network: netip.MustParsePrefix("10.1.0.0/16"),
PeerGroups: []string{"group-src", "group-dst"},
Enabled: true,
},
{
ID: "route-no-seq",
AccountSeqID: 0, // unset — should still ship (no group seq filter on routes)
Network: netip.MustParsePrefix("10.2.0.0/16"),
Enabled: true,
ID: "route-peergroup",
PublicID: "101",
NetID: "net-B",
Network: netip.MustParsePrefix("10.1.0.0/16"),
PeerGroups: []string{"group-src", "group-dst"},
Enabled: true,
},
}
full := EncodeNetworkMapEnvelope(ComponentsEnvelopeInput{Components: c}).GetFull()
require.Len(t, full.Routes, 3)
require.Len(t, full.Routes, 2)
byNetID := map[string]*proto.RouteRaw{}
for _, r := range full.Routes {
byNetID[r.NetId] = r
@@ -534,24 +513,24 @@ func TestEncodeNetworkMapEnvelope_RoutesRoundTrip(t *testing.T) {
assert.True(t, r1.PeerIndexSet, "route with peer must set peer_index_set")
require.Less(t, int(r1.PeerIndex), len(full.Peers))
assert.Equal(t, "peerc", full.Peers[r1.PeerIndex].DnsLabel)
assert.Equal(t, []int32{1}, r1.GroupIds, "group-src has AccountSeqID 1")
assert.Equal(t, []int32{2}, r1.AccessControlGroupIds, "group-dst has AccountSeqID 2")
assert.Equal(t, []string{"1"}, r1.GroupIds, "group-src has AccountSeqID 1")
assert.Equal(t, []string{"2"}, r1.AccessControlGroupIds, "group-dst has AccountSeqID 2")
assert.Empty(t, r1.PeerGroupIds)
r2 := byNetID["net-B"]
require.NotNil(t, r2)
assert.False(t, r2.PeerIndexSet, "route with peer_groups must NOT set peer_index_set")
assert.ElementsMatch(t, []int32{1, 2}, r2.PeerGroupIds)
assert.ElementsMatch(t, []string{"1", "2"}, r2.PeerGroupIds)
}
func TestEncodeNetworkMapEnvelope_RouteWithMissingPeerLeavesIndexUnset(t *testing.T) {
c := newTestComponents()
c.Routes = []*nbroute.Route{{
ID: "route-x",
AccountSeqID: 100,
Peer: "peer-not-in-components",
Network: netip.MustParsePrefix("10.0.0.0/16"),
Enabled: true,
ID: "route-x",
PublicID: "100",
Peer: "peer-not-in-components",
Network: netip.MustParsePrefix("10.0.0.0/16"),
Enabled: true,
}}
full := EncodeNetworkMapEnvelope(ComponentsEnvelopeInput{Components: c}).GetFull()
@@ -567,7 +546,7 @@ func TestEncodeNetworkMapEnvelope_ResourceOnlyPolicyShippedAndIndexed(t *testing
// is the I1 case — without unionPolicies the encoder would silently
// drop it from the wire.
resourceOnlyPolicy := &types.Policy{
ID: "pol-resource", AccountSeqID: 99, Enabled: true,
ID: "pol-resource", PublicID: "99", Enabled: true,
Rules: []*types.PolicyRule{{
ID: "rule-r", Enabled: true, Action: types.PolicyTrafficActionAccept,
Protocol: types.PolicyRuleProtocolTCP,
@@ -581,24 +560,24 @@ func TestEncodeNetworkMapEnvelope_ResourceOnlyPolicyShippedAndIndexed(t *testing
// Resource must appear in components.NetworkResources with a seq id —
// encoder uses that to translate the xid map key to uint32.
c.NetworkResources = []*resourceTypes.NetworkResource{
{ID: "resource-x", AccountSeqID: 77, Name: "res-x", Enabled: true},
{ID: "resource-x", PublicID: "77", Name: "res-x", Enabled: true},
}
full := EncodeNetworkMapEnvelope(ComponentsEnvelopeInput{Components: c}).GetFull()
require.Len(t, full.Policies, 2, "encoded policies must include both peer-traffic and resource-only")
policyByID := map[int32]*proto.PolicyCompact{}
policyIds := make([]int32, 0)
policyByID := map[string]*proto.PolicyCompact{}
policyIds := make([]string, 0)
for _, p := range full.Policies {
policyByID[p.Id] = p
policyIds = append(policyIds, p.Id)
}
require.Contains(t, policyByID, int32(10), "original peer-traffic policy id 10")
require.Contains(t, policyByID, int32(99), "resource-only policy id 99")
require.Contains(t, policyByID, "10", "original peer-traffic policy id 10")
require.Contains(t, policyByID, "99", "resource-only policy id 99")
require.Contains(t, full.ResourcePoliciesMap, int32(77))
ids := full.ResourcePoliciesMap[77].Ids
require.Contains(t, full.ResourcePoliciesMap, "77")
ids := full.ResourcePoliciesMap["77"].Ids
require.Len(t, ids, 2)
assert.ElementsMatch(t, policyIds, ids,
"resource policies map must reference both wire policy indexes")
@@ -607,7 +586,7 @@ func TestEncodeNetworkMapEnvelope_ResourceOnlyPolicyShippedAndIndexed(t *testing
func TestEncodeNetworkMapEnvelope_NameServerGroups(t *testing.T) {
c := newTestComponents()
c.NameServerGroups = []*nbdns.NameServerGroup{{
ID: "nsg-1", AccountSeqID: 50, Name: "Main", Description: "primary",
ID: "nsg-1", PublicID: "50", Name: "Main", Description: "primary",
NameServers: []nbdns.NameServer{{
IP: netip.MustParseAddr("8.8.8.8"), NSType: nbdns.UDPNameServerType, Port: 53,
}},
@@ -615,23 +594,22 @@ func TestEncodeNetworkMapEnvelope_NameServerGroups(t *testing.T) {
Primary: true, Enabled: true,
Domains: []string{"corp.example"},
}}
c.Groups["group-not-persisted"] = &types.Group{ID: "group-not-persisted", AccountSeqID: 0, Peers: []string{}}
full := EncodeNetworkMapEnvelope(ComponentsEnvelopeInput{Components: c}).GetFull()
require.Len(t, full.NameserverGroups, 1)
nsg := full.NameserverGroups[0]
assert.EqualValues(t, 50, nsg.Id)
assert.EqualValues(t, "50", nsg.Id)
assert.Equal(t, "Main", nsg.Name)
assert.True(t, nsg.Primary)
require.Len(t, nsg.Nameservers, 1)
assert.Equal(t, "8.8.8.8", nsg.Nameservers[0].IP)
assert.Equal(t, []int32{1}, nsg.GroupIds, "group-not-persisted is filtered out (AccountSeqID=0)")
assert.Equal(t, []string{"1"}, nsg.GroupIds)
}
func TestEncodeNetworkMapEnvelope_PostureFailedPeers(t *testing.T) {
c := newTestComponents()
c.PostureCheckXIDToSeq = map[string]int32{"check-1": 33}
c.PostureCheckXIDToPublicID = map[string]string{"check-1": "33"}
c.PostureFailedPeers = map[string]map[string]struct{}{
"check-1": {
"peer-a": {},
@@ -642,18 +620,18 @@ func TestEncodeNetworkMapEnvelope_PostureFailedPeers(t *testing.T) {
full := EncodeNetworkMapEnvelope(ComponentsEnvelopeInput{Components: c}).GetFull()
require.Contains(t, full.PostureFailedPeers, int32(33))
idxs := full.PostureFailedPeers[33].PeerIndexes
require.Contains(t, full.PostureFailedPeers, "33")
idxs := full.PostureFailedPeers["33"].PeerIndexes
assert.Len(t, idxs, 2, "missing peer is silently dropped (filterPostureFailedPeers guarantees presence in real data)")
}
func TestEncodeNetworkMapEnvelope_RoutersMap(t *testing.T) {
c := newTestComponents()
c.NetworkXIDToSeq = map[string]int32{"net-1": 5}
c.NetworkXIDToPublicID = map[string]string{"net-1": "5"}
c.RoutersMap = map[string]map[string]*routerTypes.NetworkRouter{
"net-1": {
"peer-c": {
ID: "router-1", AccountSeqID: 200,
ID: "router-1", PublicID: "200",
Peer: "peer-c", Masquerade: true, Metric: 10, Enabled: true,
},
},
@@ -661,11 +639,11 @@ func TestEncodeNetworkMapEnvelope_RoutersMap(t *testing.T) {
full := EncodeNetworkMapEnvelope(ComponentsEnvelopeInput{Components: c}).GetFull()
require.Contains(t, full.RoutersMap, int32(5))
entries := full.RoutersMap[5].Entries
require.Contains(t, full.RoutersMap, "5")
entries := full.RoutersMap["5"].Entries
require.Len(t, entries, 1)
e := entries[0]
assert.EqualValues(t, 200, e.Id)
assert.EqualValues(t, "200", e.Id)
assert.True(t, e.PeerIndexSet)
require.Less(t, int(e.PeerIndex), len(full.Peers))
assert.Equal(t, "peerc", full.Peers[e.PeerIndex].DnsLabel)
@@ -685,47 +663,31 @@ func TestEncodeNetworkMapEnvelope_RouterPeerNotInComponentsPeers(t *testing.T) {
DNSLabel: "peerc", Meta: nbpeer.PeerSystemMeta{WtVersion: "0.40.0"},
}
c.RouterPeers = map[string]*nbpeer.Peer{"peer-c": routerPeer}
c.NetworkXIDToSeq = map[string]int32{"net-1": 5}
c.NetworkXIDToPublicID = map[string]string{"net-1": "5"}
c.RoutersMap = map[string]map[string]*routerTypes.NetworkRouter{
"net-1": {"peer-c": {ID: "r-1", AccountSeqID: 1, Peer: "peer-c", Enabled: true}},
"net-1": {"peer-c": {ID: "r-1", PublicID: "1", Peer: "peer-c", Enabled: true}},
}
full := EncodeNetworkMapEnvelope(ComponentsEnvelopeInput{Components: c}).GetFull()
require.Contains(t, full.RoutersMap, int32(5))
require.Len(t, full.RoutersMap[5].Entries, 1)
e := full.RoutersMap[5].Entries[0]
require.Contains(t, full.RoutersMap, "5")
require.Len(t, full.RoutersMap["5"].Entries, 1)
e := full.RoutersMap["5"].Entries[0]
assert.True(t, e.PeerIndexSet, "router peer must be indexed even when not in c.Peers")
}
func TestEncodeNetworkMapEnvelope_DNSSettingsFiltersUnpersistedGroups(t *testing.T) {
c := newTestComponents()
c.DNSSettings = &types.DNSSettings{
DisabledManagementGroups: []string{"group-src", "group-missing", "group-no-seq"},
}
c.Groups["group-no-seq"] = &types.Group{ID: "group-no-seq", AccountSeqID: 0}
full := EncodeNetworkMapEnvelope(ComponentsEnvelopeInput{Components: c}).GetFull()
require.NotNil(t, full.DnsSettings)
assert.Equal(t, []int32{1}, full.DnsSettings.DisabledManagementGroupIds,
"only group-src (AccountSeqID=1) survives — missing and unpersisted are dropped")
}
func TestEncodeNetworkMapEnvelope_GroupIDToUserIDs(t *testing.T) {
c := newTestComponents()
c.GroupIDToUserIDs = map[string][]string{
"group-src": {"user-1", "user-2"},
"group-no-seq": {"user-3"}, // group not persisted → drop
"group-missing": {"user-4"}, // group not in components → drop
}
c.Groups["group-no-seq"] = &types.Group{ID: "group-no-seq", AccountSeqID: 0}
full := EncodeNetworkMapEnvelope(ComponentsEnvelopeInput{Components: c}).GetFull()
require.Len(t, full.GroupIdToUserIds, 1, "only persisted+present groups survive")
require.Contains(t, full.GroupIdToUserIds, int32(1))
assert.ElementsMatch(t, []string{"user-1", "user-2"}, full.GroupIdToUserIds[1].UserIds)
require.Len(t, full.GroupIdToUserIds, 1, "only present groups survive")
require.Contains(t, full.GroupIdToUserIds, "1")
assert.ElementsMatch(t, []string{"user-1", "user-2"}, full.GroupIdToUserIds["1"].UserIds)
}
func TestToProxyPatch_EmptyInputReturnsNil(t *testing.T) {
management/server/account.go
+1 -5
View File
@@ -1649,11 +1649,7 @@ func (am *DefaultAccountManager) SyncUserJWTGroups(ctx context.Context, userAuth
}
for _, g := range newGroupsToCreate {
seq, err := transaction.AllocateAccountSeqID(ctx, userAuth.AccountId, types.AccountSeqEntityGroup)
if err != nil {
return fmt.Errorf("error allocating group seq id: %w", err)
}
g.AccountSeqID = seq
g.PublicID = xid.New().String()
}
if err = transaction.CreateGroups(ctx, userAuth.AccountId, newGroupsToCreate); err != nil {
management/server/account_test.go
+1 -1
View File
@@ -3179,7 +3179,7 @@ func TestAccount_SetJWTGroups(t *testing.T) {
}
}
require.NotNil(t, newJWTGroup, "JIT-created JWT group not found")
assert.NotZero(t, newJWTGroup.AccountSeqID, "JIT-created JWT group must have a non-zero AccountSeqID")
assert.NotEqual(t, "", newJWTGroup.PublicID, "JIT-created JWT group must have a non-empty PublicID")
})
t.Run("remove all JWT groups when list is empty", func(t *testing.T) {
management/server/group.go
+4 -13
View File
@@ -93,11 +93,7 @@ func (am *DefaultAccountManager) CreateGroup(ctx context.Context, accountID, use
events := am.prepareGroupEvents(ctx, transaction, accountID, userID, newGroup)
eventsToStore = append(eventsToStore, events...)
seq, err := transaction.AllocateAccountSeqID(ctx, accountID, types.AccountSeqEntityGroup)
if err != nil {
return status.Errorf(status.Internal, "failed to allocate group seq id: %v", err)
}
newGroup.AccountSeqID = seq
newGroup.PublicID = xid.New().String()
if err := transaction.CreateGroup(ctx, newGroup); err != nil {
return status.Errorf(status.Internal, "failed to create group: %v", err)
@@ -164,7 +160,7 @@ func (am *DefaultAccountManager) UpdateGroup(ctx context.Context, accountID, use
return err
}
newGroup.AccountSeqID = oldGroup.AccountSeqID
newGroup.PublicID = oldGroup.PublicID
if err = transaction.UpdateGroup(ctx, newGroup); err != nil {
return err
@@ -243,12 +239,7 @@ func (am *DefaultAccountManager) CreateGroups(ctx context.Context, accountID, us
}
newGroup.AccountID = accountID
seq, err := transaction.AllocateAccountSeqID(ctx, accountID, types.AccountSeqEntityGroup)
if err != nil {
return err
}
newGroup.AccountSeqID = seq
newGroup.PublicID = xid.New().String()
if err = transaction.CreateGroup(ctx, newGroup); err != nil {
return err
@@ -345,7 +336,7 @@ func (am *DefaultAccountManager) updateSingleGroup(ctx context.Context, accountI
if err != nil {
return err
}
newGroup.AccountSeqID = oldGroup.AccountSeqID
newGroup.PublicID = oldGroup.PublicID
if err := transaction.UpdateGroup(ctx, newGroup); err != nil {
return err
management/server/migration/account_seq.go
-156
View File
@@ -1,156 +0,0 @@
package migration
import (
"context"
"fmt"
log "github.com/sirupsen/logrus"
"gorm.io/gorm"
"github.com/netbirdio/netbird/management/server/types"
)
// BackfillAccountSeqIDs assigns a deterministic per-account sequential id to all
// rows of `model` whose account_seq_id is zero, then seeds account_seq_counters
// with the next free id per account. Idempotent: safe to re-run; both steps
// no-op once everything is consistent.
//
// Implemented as two table-wide SQL statements with window functions, one
// transaction. Backfilling 246k rows across 154k accounts on Postgres takes
// well under a second instead of the per-account-loop ~2 minutes.
//
// orderColumn is the column to use when assigning the deterministic ordering
// (typically the primary-key string id).
func BackfillAccountSeqIDs[T any](
ctx context.Context,
db *gorm.DB,
entity types.AccountSeqEntity,
orderColumn string,
) error {
var model T
if !db.Migrator().HasTable(&model) {
log.WithContext(ctx).Debugf("backfill seq id: table for %T missing, skip", model)
return nil
}
stmt := &gorm.Statement{DB: db}
if err := stmt.Parse(&model); err != nil {
return fmt.Errorf("parse model: %w", err)
}
table := quoteIdent(db, stmt.Schema.Table)
orderCol := quoteIdent(db, orderColumn)
return db.Transaction(func(tx *gorm.DB) error {
var pending int64
if err := tx.Raw(
fmt.Sprintf("SELECT count(*) FROM %s WHERE account_seq_id IS NULL OR account_seq_id = 0", table),
).Scan(&pending).Error; err != nil {
return fmt.Errorf("count pending on %s: %w", table, err)
}
if pending > 0 {
log.WithContext(ctx).Infof("backfill seq id: %s — %d rows pending", table, pending)
if err := backfillRankSQL(tx, table, orderCol); err != nil {
return fmt.Errorf("rank %s: %w", table, err)
}
}
if err := seedCountersSQL(tx, table, entity); err != nil {
return fmt.Errorf("seed counters for %s: %w", entity, err)
}
return nil
})
}
func quoteIdent(db *gorm.DB, name string) string {
switch db.Dialector.Name() {
case "mysql":
return "`" + name + "`"
case "postgres":
return `"` + name + `"`
default:
return name
}
}
func backfillRankSQL(db *gorm.DB, table, orderCol string) error {
dialect := db.Dialector.Name()
var sql string
switch dialect {
case "postgres", "sqlite":
sql = fmt.Sprintf(`
WITH max_seq AS (
SELECT account_id, COALESCE(MAX(account_seq_id), 0) AS max_seq
FROM %s
GROUP BY account_id
),
ranked AS (
SELECT p.id,
m.max_seq + ROW_NUMBER() OVER (PARTITION BY p.account_id ORDER BY p.%s) AS new_seq
FROM %s p
JOIN max_seq m ON p.account_id = m.account_id
WHERE p.account_seq_id IS NULL OR p.account_seq_id = 0
)
UPDATE %s SET account_seq_id = ranked.new_seq
FROM ranked
WHERE %s.id = ranked.id
`, table, orderCol, table, table, table)
case "mysql":
sql = fmt.Sprintf(`
UPDATE %s p
JOIN (
SELECT account_id, COALESCE(MAX(account_seq_id), 0) AS max_seq
FROM %s
GROUP BY account_id
) m ON p.account_id = m.account_id
JOIN (
SELECT id, ROW_NUMBER() OVER (PARTITION BY account_id ORDER BY %s) AS rn
FROM %s
WHERE account_seq_id IS NULL OR account_seq_id = 0
) r ON p.id = r.id
SET p.account_seq_id = m.max_seq + r.rn
`, table, table, orderCol, table)
default:
return fmt.Errorf("unsupported dialect: %s", dialect)
}
return db.Exec(sql).Error
}
func seedCountersSQL(db *gorm.DB, table string, entity types.AccountSeqEntity) error {
dialect := db.Dialector.Name()
var sql string
switch dialect {
case "postgres":
sql = fmt.Sprintf(`
INSERT INTO account_seq_counters (account_id, entity, next_id)
SELECT account_id, ?, MAX(account_seq_id) + 1
FROM %s
WHERE account_seq_id IS NOT NULL AND account_seq_id > 0
GROUP BY account_id
ON CONFLICT (account_id, entity) DO UPDATE
SET next_id = GREATEST(account_seq_counters.next_id, EXCLUDED.next_id)
`, table)
case "sqlite":
sql = fmt.Sprintf(`
INSERT INTO account_seq_counters (account_id, entity, next_id)
SELECT account_id, ?, MAX(account_seq_id) + 1
FROM %s
WHERE account_seq_id IS NOT NULL AND account_seq_id > 0
GROUP BY account_id
ON CONFLICT (account_id, entity) DO UPDATE
SET next_id = max(account_seq_counters.next_id, excluded.next_id)
`, table)
case "mysql":
sql = fmt.Sprintf(`
INSERT INTO account_seq_counters (account_id, entity, next_id)
SELECT account_id, ?, MAX(account_seq_id) + 1
FROM %s
WHERE account_seq_id IS NOT NULL AND account_seq_id > 0
GROUP BY account_id
ON DUPLICATE KEY UPDATE next_id = GREATEST(next_id, VALUES(next_id))
`, table)
default:
return fmt.Errorf("unsupported dialect: %s", dialect)
}
return db.Exec(sql, string(entity)).Error
}
management/server/nameserver.go
+2 -6
View File
@@ -67,11 +67,7 @@ func (am *DefaultAccountManager) CreateNameServerGroup(ctx context.Context, acco
return err
}
seq, err := transaction.AllocateAccountSeqID(ctx, accountID, types.AccountSeqEntityNameserverGroup)
if err != nil {
return err
}
newNSGroup.AccountSeqID = seq
newNSGroup.PublicID = xid.New().String()
if err = transaction.SaveNameServerGroup(ctx, newNSGroup); err != nil {
return err
@@ -122,7 +118,7 @@ func (am *DefaultAccountManager) SaveNameServerGroup(ctx context.Context, accoun
return err
}
nsGroupToSave.AccountSeqID = oldNSGroup.AccountSeqID
nsGroupToSave.PublicID = oldNSGroup.PublicID
if err = transaction.SaveNameServerGroup(ctx, nsGroupToSave); err != nil {
return err
management/server/networks/manager.go
+2 -7
View File
@@ -16,7 +16,6 @@ import (
"github.com/netbirdio/netbird/management/server/permissions/modules"
"github.com/netbirdio/netbird/management/server/permissions/operations"
"github.com/netbirdio/netbird/management/server/store"
serverTypes "github.com/netbirdio/netbird/management/server/types"
"github.com/netbirdio/netbird/shared/management/status"
)
@@ -73,11 +72,7 @@ func (m *managerImpl) CreateNetwork(ctx context.Context, userID string, network
network.ID = xid.New().String()
err = m.store.ExecuteInTransaction(ctx, func(transaction store.Store) error {
seq, err := transaction.AllocateAccountSeqID(ctx, network.AccountID, serverTypes.AccountSeqEntityNetwork)
if err != nil {
return fmt.Errorf("failed to allocate network seq id: %w", err)
}
network.AccountSeqID = seq
network.PublicID = xid.New().String()
if err := transaction.SaveNetwork(ctx, network); err != nil {
return fmt.Errorf("failed to save network: %w", err)
@@ -119,7 +114,7 @@ func (m *managerImpl) UpdateNetwork(ctx context.Context, userID string, network
if err != nil {
return fmt.Errorf("failed to get network: %w", err)
}
network.AccountSeqID = existing.AccountSeqID
network.PublicID = existing.PublicID
if err := transaction.SaveNetwork(ctx, network); err != nil {
return fmt.Errorf("failed to save network: %w", err)
management/server/networks/manager_test.go
+7 -7
View File
@@ -259,7 +259,7 @@ func Test_UpdateNetworkFailsWithPermissionDenied(t *testing.T) {
// Test_CreateNetworkAllocatesSeqID verifies that CreateNetwork sets a
// non-zero AccountSeqID on the persisted network (allocated through the
// account_seq_counters table).
func Test_CreateNetworkAllocatesSeqID(t *testing.T) {
func Test_CreateNetworkSetsPublicId(t *testing.T) {
ctx := context.Background()
const accountID = "testAccountId"
const userID = "testAdminId"
@@ -280,13 +280,13 @@ func Test_CreateNetworkAllocatesSeqID(t *testing.T) {
Name: "seq-allocation-test",
})
require.NoError(t, err)
require.NotZero(t, created.AccountSeqID, "CreateNetwork must allocate a non-zero AccountSeqID")
require.NotEqual(t, "", created.PublicID, "CreateNetwork must allocate a non-zero AccountSeqID")
}
// Test_UpdateNetworkPreservesSeqID verifies UpdateNetwork does not reset
// AccountSeqID even when the caller passes a zero value (the shape REST
// handlers produce because the field is `json:"-"`).
func Test_UpdateNetworkPreservesSeqID(t *testing.T) {
func Test_UpdateNetworkPreservesPublicId(t *testing.T) {
ctx := context.Background()
const accountID = "testAccountId"
const userID = "testAdminId"
@@ -307,21 +307,21 @@ func Test_UpdateNetworkPreservesSeqID(t *testing.T) {
Name: "seq-preserve-original",
})
require.NoError(t, err)
originalSeq := created.AccountSeqID
require.NotZero(t, originalSeq)
originalPublicId := created.PublicID
require.NotZero(t, originalPublicId)
update := &types.Network{
AccountID: accountID,
ID: created.ID,
Name: "seq-preserve-renamed",
}
require.Zero(t, update.AccountSeqID, "incoming struct must mirror an HTTP handler shape")
require.Equal(t, "", update.PublicID, "incoming struct must mirror an HTTP handler shape")
_, err = manager.UpdateNetwork(ctx, userID, update)
require.NoError(t, err)
got, err := manager.GetNetwork(ctx, accountID, userID, created.ID)
require.NoError(t, err)
require.Equal(t, originalSeq, got.AccountSeqID, "AccountSeqID must survive UpdateNetwork")
require.Equal(t, originalPublicId, got.PublicID, "PublicID must survive UpdateNetwork")
require.Equal(t, "seq-preserve-renamed", got.Name)
}
management/server/networks/resources/manager.go
+3 -6
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"github.com/rs/xid"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/management/internals/modules/reverseproxy/service"
@@ -146,11 +147,7 @@ func (m *managerImpl) createResourceInTransaction(ctx context.Context, transacti
return nil, nil, fmt.Errorf("failed to get network: %w", err)
}
seq, err := transaction.AllocateAccountSeqID(ctx, resource.AccountID, nbtypes.AccountSeqEntityNetworkResource)
if err != nil {
return nil, nil, fmt.Errorf("failed to allocate network resource seq id: %w", err)
}
resource.AccountSeqID = seq
resource.PublicID = xid.New().String()
if err = transaction.SaveNetworkResource(ctx, resource); err != nil {
return nil, nil, fmt.Errorf("failed to save network resource: %w", err)
@@ -251,7 +248,7 @@ func (m *managerImpl) UpdateResource(ctx context.Context, userID string, resourc
if err != nil {
return fmt.Errorf("failed to get network resource: %w", err)
}
resource.AccountSeqID = oldResource.AccountSeqID
resource.PublicID = oldResource.PublicID
oldGroups, err := m.groupsManager.GetResourceGroupsInTransaction(ctx, transaction, store.LockingStrengthNone, resource.AccountID, resource.ID)
if err != nil {
management/server/networks/resources/types/resource.go
+24 -26
View File
@@ -29,20 +29,18 @@ func (p NetworkResourceType) String() string {
}
type NetworkResource struct {
ID string `gorm:"primaryKey"`
NetworkID string `gorm:"index"`
AccountID string `gorm:"index"`
// AccountSeqID is a per-account monotonically increasing identifier used as the
// compact wire id when sending NetworkMap components to capable peers.
AccountSeqID int32 `json:"-" gorm:"not null;default:0"`
Name string
Description string
Type NetworkResourceType
Address string `gorm:"-"`
GroupIDs []string `gorm:"-"`
Domain string
Prefix netip.Prefix `gorm:"serializer:json"`
Enabled bool
ID string `gorm:"primaryKey"`
NetworkID string `gorm:"index"`
AccountID string `gorm:"index"`
PublicID string `json:"-"`
Name string
Description string
Type NetworkResourceType
Address string `gorm:"-"`
GroupIDs []string `gorm:"-"`
Domain string
Prefix netip.Prefix `gorm:"serializer:json"`
Enabled bool
}
func NewNetworkResource(accountID, networkID, name, description, address string, groupIDs []string, enabled bool) (*NetworkResource, error) {
@@ -96,18 +94,18 @@ func (n *NetworkResource) FromAPIRequest(req *api.NetworkResourceRequest) {
func (n *NetworkResource) Copy() *NetworkResource {
return &NetworkResource{
ID: n.ID,
AccountID: n.AccountID,
NetworkID: n.NetworkID,
AccountSeqID: n.AccountSeqID,
Name: n.Name,
Description: n.Description,
Type: n.Type,
Address: n.Address,
Domain: n.Domain,
Prefix: n.Prefix,
GroupIDs: n.GroupIDs,
Enabled: n.Enabled,
ID: n.ID,
AccountID: n.AccountID,
NetworkID: n.NetworkID,
PublicID: n.PublicID,
Name: n.Name,
Description: n.Description,
Type: n.Type,
Address: n.Address,
Domain: n.Domain,
Prefix: n.Prefix,
GroupIDs: n.GroupIDs,
Enabled: n.Enabled,
}
}
management/server/networks/routers/manager.go
+3 -8
View File
@@ -16,7 +16,6 @@ import (
"github.com/netbirdio/netbird/management/server/permissions/modules"
"github.com/netbirdio/netbird/management/server/permissions/operations"
"github.com/netbirdio/netbird/management/server/store"
serverTypes "github.com/netbirdio/netbird/management/server/types"
"github.com/netbirdio/netbird/shared/management/status"
)
@@ -105,11 +104,7 @@ func (m *managerImpl) CreateRouter(ctx context.Context, userID string, router *t
router.ID = xid.New().String()
seq, err := transaction.AllocateAccountSeqID(ctx, router.AccountID, serverTypes.AccountSeqEntityNetworkRouter)
if err != nil {
return fmt.Errorf("failed to allocate network router seq id: %w", err)
}
router.AccountSeqID = seq
router.PublicID = xid.New().String()
err = transaction.CreateNetworkRouter(ctx, router)
if err != nil {
@@ -206,10 +201,10 @@ func (m *managerImpl) updateRouterInTransaction(ctx context.Context, transaction
return nil, nil, affectedpeers.Change{}, status.NewRouterNotPartOfNetworkError(router.ID, router.NetworkID)
}
// Preserve AccountSeqID from the existing router so the upstream
// Preserve PublicID from the existing router so the upstream
// UpdateNetworkRouter (which does Updates(router) with Select("*"))
// doesn't clobber it with the request's zero value.
router.AccountSeqID = existing.AccountSeqID
router.PublicID = existing.PublicID
if err = transaction.UpdateNetworkRouter(ctx, router); err != nil {
return nil, nil, affectedpeers.Change{}, fmt.Errorf("failed to update network router: %w", err)
management/server/networks/routers/types/router.go
+18 -20
View File
@@ -10,17 +10,15 @@ import (
)
type NetworkRouter struct {
ID string `gorm:"primaryKey"`
NetworkID string `gorm:"index"`
AccountID string `gorm:"index"`
// AccountSeqID is a per-account monotonically increasing identifier used as the
// compact wire id when sending NetworkMap components to capable peers.
AccountSeqID int32 `json:"-" gorm:"not null;default:0"`
Peer string
PeerGroups []string `gorm:"serializer:json"`
Masquerade bool
Metric int
Enabled bool
ID string `gorm:"primaryKey"`
NetworkID string `gorm:"index"`
AccountID string `gorm:"index"`
PublicID string `json:"-"`
Peer string
PeerGroups []string `gorm:"serializer:json"`
Masquerade bool
Metric int
Enabled bool
}
func NewNetworkRouter(accountID string, networkID string, peer string, peerGroups []string, masquerade bool, metric int, enabled bool) (*NetworkRouter, error) {
@@ -81,15 +79,15 @@ func (n *NetworkRouter) FromAPIRequest(req *api.NetworkRouterRequest) {
func (n *NetworkRouter) Copy() *NetworkRouter {
return &NetworkRouter{
ID: n.ID,
NetworkID: n.NetworkID,
AccountID: n.AccountID,
AccountSeqID: n.AccountSeqID,
Peer: n.Peer,
PeerGroups: n.PeerGroups,
Masquerade: n.Masquerade,
Metric: n.Metric,
Enabled: n.Enabled,
ID: n.ID,
NetworkID: n.NetworkID,
AccountID: n.AccountID,
PublicID: n.PublicID,
Peer: n.Peer,
PeerGroups: n.PeerGroups,
Masquerade: n.Masquerade,
Metric: n.Metric,
Enabled: n.Enabled,
}
}
management/server/networks/types/network.go
+6 -15
View File
@@ -10,21 +10,12 @@ type Network struct {
ID string `gorm:"primaryKey"`
AccountID string `gorm:"index"`
// AccountSeqID is a per-account monotonically increasing identifier used as the
// compact wire id when sending NetworkMap components to capable peers.
AccountSeqID int32 `json:"-" gorm:"not null;default:0"`
PublicID string `json:"-"`
Name string
Description string
}
// HasSeqID reports whether the network has been persisted long enough to have
// a per-account sequence id allocated. Wire encoders that key off AccountSeqID
// must skip networks that return false here.
func (n *Network) HasSeqID() bool {
return n != nil && n.AccountSeqID != 0
}
func NewNetwork(accountId, name, description string) *Network {
return &Network{
ID: xid.New().String(),
@@ -56,11 +47,11 @@ func (n *Network) FromAPIRequest(req *api.NetworkRequest) {
// Copy returns a copy of a network.
func (n *Network) Copy() *Network {
return &Network{
ID: n.ID,
AccountID: n.AccountID,
AccountSeqID: n.AccountSeqID,
Name: n.Name,
Description: n.Description,
ID: n.ID,
AccountID: n.AccountID,
PublicID: n.PublicID,
Name: n.Name,
Description: n.Description,
}
}
management/server/policy.go
+2 -7
View File
@@ -67,18 +67,13 @@ func (am *DefaultAccountManager) SavePolicy(ctx context.Context, accountID, user
action = activity.PolicyUpdated
policy.AccountSeqID = existingPolicy.AccountSeqID
policy.PublicID = existingPolicy.PublicID
if err = transaction.SavePolicy(ctx, policy); err != nil {
return err
}
} else {
seq, err := transaction.AllocateAccountSeqID(ctx, accountID, types.AccountSeqEntityPolicy)
if err != nil {
return err
}
policy.AccountSeqID = seq
policy.PublicID = xid.New().String()
if err = transaction.CreatePolicy(ctx, policy); err != nil {
return err
}
management/server/posture/checks.go
+7 -16
View File
@@ -49,9 +49,7 @@ type Checks struct {
// AccountID is a reference to the Account that this object belongs
AccountID string `json:"-" gorm:"index"`
// AccountSeqID is a per-account monotonically increasing identifier used as the
// compact wire id when sending NetworkMap components to capable peers.
AccountSeqID int32 `json:"-" gorm:"not null;default:0"`
PublicID string `json:"-"`
// Checks is a set of objects that perform the actual checks
Checks ChecksDefinition `gorm:"serializer:json"`
@@ -97,13 +95,6 @@ func verdictChanged(ctx context.Context, check Check, oldPeer, newPeer nbpeer.Pe
return changed
}
// HasSeqID reports whether the posture check has been persisted long enough
// to have a per-account sequence id allocated. Wire encoders that key off
// AccountSeqID must skip checks that return false here.
func (pc *Checks) HasSeqID() bool {
return pc != nil && pc.AccountSeqID != 0
}
// ChecksDefinition contains definition of actual check
type ChecksDefinition struct {
NBVersionCheck *NBVersionCheck `json:",omitempty"`
@@ -174,12 +165,12 @@ func (*Checks) TableName() string {
// Copy returns a copy of a posture checks.
func (pc *Checks) Copy() *Checks {
checks := &Checks{
ID: pc.ID,
Name: pc.Name,
Description: pc.Description,
AccountID: pc.AccountID,
AccountSeqID: pc.AccountSeqID,
Checks: pc.Checks.Copy(),
ID: pc.ID,
Name: pc.Name,
Description: pc.Description,
AccountID: pc.AccountID,
PublicID: pc.PublicID,
Checks: pc.Checks.Copy(),
}
return checks
}
management/server/posture_checks.go
+2 -7
View File
@@ -12,7 +12,6 @@ import (
"github.com/netbirdio/netbird/management/server/permissions/operations"
"github.com/netbirdio/netbird/management/server/posture"
"github.com/netbirdio/netbird/management/server/store"
"github.com/netbirdio/netbird/management/server/types"
"github.com/netbirdio/netbird/shared/management/status"
)
@@ -57,15 +56,11 @@ func (am *DefaultAccountManager) SavePostureChecks(ctx context.Context, accountI
if err != nil {
return err
}
postureChecks.AccountSeqID = existing.AccountSeqID
postureChecks.PublicID = existing.PublicID
action = activity.PostureCheckUpdated
} else {
seq, err := transaction.AllocateAccountSeqID(ctx, accountID, types.AccountSeqEntityPostureCheck)
if err != nil {
return err
}
postureChecks.AccountSeqID = seq
postureChecks.PublicID = xid.New().String()
}
postureChecks.AccountID = accountID
management/server/posture_checks_test.go
+5 -5
View File
@@ -581,7 +581,7 @@ func TestSavePostureChecks_AllocatesSeqIDOnCreate(t *testing.T) {
},
}, true)
require.NoError(t, err)
require.NotZero(t, created.AccountSeqID, "SavePostureChecks on create must allocate a non-zero AccountSeqID")
require.NotEqual(t, "", created.PublicID, "SavePostureChecks on create must create PublicID")
}
// TestSavePostureChecks_PreservesSeqIDOnUpdate verifies the update path does
@@ -601,8 +601,8 @@ func TestSavePostureChecks_PreservesSeqIDOnUpdate(t *testing.T) {
},
}, true)
require.NoError(t, err)
originalSeq := created.AccountSeqID
require.NotZero(t, originalSeq)
originalPublicID := created.PublicID
require.NotEqual(t, "", originalPublicID)
update := &posture.Checks{
ID: created.ID,
@@ -611,13 +611,13 @@ func TestSavePostureChecks_PreservesSeqIDOnUpdate(t *testing.T) {
NBVersionCheck: &posture.NBVersionCheck{MinVersion: "0.27.0"},
},
}
require.Zero(t, update.AccountSeqID, "incoming struct must mirror an HTTP handler shape")
require.Equal(t, "", update.PublicID, "incoming struct must mirror an HTTP handler shape")
_, err = am.SavePostureChecks(context.Background(), account.Id, adminUserID, update, false)
require.NoError(t, err)
got, err := am.GetPostureChecks(context.Background(), account.Id, created.ID, adminUserID)
require.NoError(t, err)
require.Equal(t, originalSeq, got.AccountSeqID, "AccountSeqID must survive SavePostureChecks update")
require.Equal(t, originalPublicID, got.PublicID, "PublicID must survive SavePostureChecks update")
require.Equal(t, "seq-preserve-renamed", got.Name)
}
management/server/route.go
+2 -6
View File
@@ -175,11 +175,7 @@ func (am *DefaultAccountManager) CreateRoute(ctx context.Context, accountID stri
return err
}
seq, err := transaction.AllocateAccountSeqID(ctx, accountID, types.AccountSeqEntityRoute)
if err != nil {
return err
}
newRoute.AccountSeqID = seq
newRoute.PublicID = xid.New().String()
if err = transaction.SaveRoute(ctx, newRoute); err != nil {
return err
@@ -228,7 +224,7 @@ func (am *DefaultAccountManager) SaveRoute(ctx context.Context, accountID, userI
}
routeToSave.AccountID = accountID
routeToSave.AccountSeqID = oldRoute.AccountSeqID
routeToSave.PublicID = oldRoute.PublicID
if err = transaction.SaveRoute(ctx, routeToSave); err != nil {
return err
management/server/store/account_seq_test.go
-506
View File
@@ -1,506 +0,0 @@
package store
import (
"context"
"errors"
"net/netip"
"testing"
"github.com/stretchr/testify/require"
nbdns "github.com/netbirdio/netbird/dns"
resourceTypes "github.com/netbirdio/netbird/management/server/networks/resources/types"
routerTypes "github.com/netbirdio/netbird/management/server/networks/routers/types"
networkTypes "github.com/netbirdio/netbird/management/server/networks/types"
"github.com/netbirdio/netbird/management/server/posture"
"github.com/netbirdio/netbird/management/server/types"
"github.com/netbirdio/netbird/route"
)
var errRollback = errors.New("intentional rollback")
func TestAllocateAccountSeqID_SequentialPerAccount(t *testing.T) {
store, cleanup, err := NewTestStoreFromSQL(context.Background(), "", t.TempDir())
t.Cleanup(cleanup)
require.NoError(t, err)
ctx := context.Background()
const accA = "acc-a"
const accB = "acc-b"
require.NoError(t, store.ExecuteInTransaction(ctx, func(tx Store) error {
got, err := tx.AllocateAccountSeqID(ctx, accA, types.AccountSeqEntityPolicy)
require.NoError(t, err)
require.Equal(t, uint32(1), got)
got, err = tx.AllocateAccountSeqID(ctx, accA, types.AccountSeqEntityPolicy)
require.NoError(t, err)
require.Equal(t, uint32(2), got)
got, err = tx.AllocateAccountSeqID(ctx, accB, types.AccountSeqEntityPolicy)
require.NoError(t, err)
require.Equal(t, uint32(1), got, "different account starts from 1")
got, err = tx.AllocateAccountSeqID(ctx, accA, types.AccountSeqEntityGroup)
require.NoError(t, err)
require.Equal(t, uint32(1), got, "different entity starts from 1")
return nil
}))
require.NoError(t, store.ExecuteInTransaction(ctx, func(tx Store) error {
got, err := tx.AllocateAccountSeqID(ctx, accA, types.AccountSeqEntityPolicy)
require.NoError(t, err)
require.Equal(t, uint32(3), got, "counter persists across transactions")
return nil
}))
}
func TestPolicyBackfill_AssignsSeqIDsToExistingPolicies(t *testing.T) {
store, cleanup, err := NewTestStoreFromSQL(context.Background(), "../testdata/store.sql", t.TempDir())
t.Cleanup(cleanup)
require.NoError(t, err)
ctx := context.Background()
const accountID = "bf1c8084-ba50-4ce7-9439-34653001fc3b"
policies, err := store.GetAccountPolicies(ctx, LockingStrengthNone, accountID)
require.NoError(t, err)
require.NotEmpty(t, policies, "test fixture must have policies")
seen := make(map[uint32]bool)
for _, p := range policies {
require.NotZero(t, p.AccountSeqID, "policy %s must have a non-zero AccountSeqID after migration", p.ID)
require.False(t, seen[p.AccountSeqID], "duplicate AccountSeqID %d in account %s", p.AccountSeqID, accountID)
seen[p.AccountSeqID] = true
}
}
func TestPolicyUpdate_PreservesSeqID(t *testing.T) {
store, cleanup, err := NewTestStoreFromSQL(context.Background(), "../testdata/store.sql", t.TempDir())
t.Cleanup(cleanup)
require.NoError(t, err)
ctx := context.Background()
const accountID = "bf1c8084-ba50-4ce7-9439-34653001fc3b"
const policyID = "cs1tnh0hhcjnqoiuebf0"
original, err := store.GetPolicyByID(ctx, LockingStrengthNone, accountID, policyID)
require.NoError(t, err)
originalSeq := original.AccountSeqID
require.NotZero(t, originalSeq, "fixture must have non-zero AccountSeqID after backfill")
updated := &types.Policy{
ID: policyID,
AccountID: accountID,
Name: "renamed",
Enabled: false,
Rules: original.Rules,
}
require.Zero(t, updated.AccountSeqID, "incoming struct should have zero AccountSeqID like an HTTP handler would")
require.NoError(t, store.SavePolicy(ctx, updated))
got, err := store.GetPolicyByID(ctx, LockingStrengthNone, accountID, policyID)
require.NoError(t, err)
require.Equal(t, originalSeq, got.AccountSeqID, "AccountSeqID must not be reset by update path")
require.Equal(t, "renamed", got.Name)
}
func TestGroupUpdate_PreservesSeqID(t *testing.T) {
store, cleanup, err := NewTestStoreFromSQL(context.Background(), "../testdata/store.sql", t.TempDir())
t.Cleanup(cleanup)
require.NoError(t, err)
ctx := context.Background()
const accountID = "bf1c8084-ba50-4ce7-9439-34653001fc3b"
groups, err := store.GetAccountGroups(ctx, LockingStrengthNone, accountID)
require.NoError(t, err)
require.NotEmpty(t, groups)
original := groups[0]
originalSeq := original.AccountSeqID
require.NotZero(t, originalSeq)
updated := &types.Group{
ID: original.ID,
AccountID: accountID,
Name: "renamed",
Issued: original.Issued,
}
require.Zero(t, updated.AccountSeqID)
require.NoError(t, store.UpdateGroup(ctx, updated))
got, err := store.GetGroupByID(ctx, LockingStrengthNone, accountID, original.ID)
require.NoError(t, err)
require.Equal(t, originalSeq, got.AccountSeqID, "AccountSeqID must not be reset by UpdateGroup")
require.Equal(t, "renamed", got.Name)
}
func TestSaveAccount_AllocatesSeqIDsForDefaultGroupAndPolicy(t *testing.T) {
store, cleanup, err := NewTestStoreFromSQL(context.Background(), "", t.TempDir())
t.Cleanup(cleanup)
require.NoError(t, err)
ctx := context.Background()
const accountID = "save-account-seqid-test"
account := &types.Account{
Id: accountID,
CreatedBy: "user1",
Domain: "example.test",
DNSSettings: types.DNSSettings{},
Settings: &types.Settings{},
Network: &types.Network{
Identifier: "net-test",
},
Users: map[string]*types.User{
"user1": {Id: "user1", AccountID: accountID, Role: types.UserRoleOwner},
},
}
require.NoError(t, account.AddAllGroup(false), "AddAllGroup should populate default Group + Policy")
require.Len(t, account.Groups, 1, "default 'All' group must be present")
require.Len(t, account.Policies, 1, "default policy must be present")
for _, g := range account.Groups {
require.Zero(t, g.AccountSeqID, "default group must start with seq=0")
}
require.Zero(t, account.Policies[0].AccountSeqID, "default policy must start with seq=0")
require.NoError(t, store.SaveAccount(ctx, account))
groups, err := store.GetAccountGroups(ctx, LockingStrengthNone, accountID)
require.NoError(t, err)
require.Len(t, groups, 1)
require.NotZerof(t, groups[0].AccountSeqID, "default group must have seq>0 after SaveAccount")
policies, err := store.GetAccountPolicies(ctx, LockingStrengthNone, accountID)
require.NoError(t, err)
require.Len(t, policies, 1)
require.NotZerof(t, policies[0].AccountSeqID, "default policy must have seq>0 after SaveAccount")
require.ErrorIs(t, store.ExecuteInTransaction(ctx, func(tx Store) error {
next, err := tx.AllocateAccountSeqID(ctx, accountID, types.AccountSeqEntityGroup)
require.NoError(t, err)
require.Equal(t, groups[0].AccountSeqID+1, next, "next group seq must be max+1")
next, err = tx.AllocateAccountSeqID(ctx, accountID, types.AccountSeqEntityPolicy)
require.NoError(t, err)
require.Equal(t, policies[0].AccountSeqID+1, next, "next policy seq must be max+1")
return errRollback
}), errRollback)
}
func TestSaveAccount_PreservesExistingSeqIDs(t *testing.T) {
store, cleanup, err := NewTestStoreFromSQL(context.Background(), "../testdata/store.sql", t.TempDir())
t.Cleanup(cleanup)
require.NoError(t, err)
ctx := context.Background()
const accountID = "bf1c8084-ba50-4ce7-9439-34653001fc3b"
account, err := store.GetAccount(ctx, accountID)
require.NoError(t, err)
groupSeqs := make(map[string]uint32)
policySeqs := make(map[string]uint32)
routeSeqs := make(map[route.ID]uint32)
nsgSeqs := make(map[string]uint32)
resourceSeqs := make(map[string]uint32)
routerSeqs := make(map[string]uint32)
networkSeqs := make(map[string]uint32)
for _, g := range account.Groups {
require.NotZero(t, g.AccountSeqID, "fixture group must have seq>0 after backfill")
groupSeqs[g.ID] = g.AccountSeqID
}
for _, p := range account.Policies {
require.NotZero(t, p.AccountSeqID, "fixture policy must have seq>0")
policySeqs[p.ID] = p.AccountSeqID
}
for _, r := range account.Routes {
require.NotZero(t, r.AccountSeqID, "fixture route must have seq>0")
routeSeqs[r.ID] = r.AccountSeqID
}
for _, n := range account.NameServerGroups {
require.NotZero(t, n.AccountSeqID, "fixture name_server_group must have seq>0")
nsgSeqs[n.ID] = n.AccountSeqID
}
for _, nr := range account.NetworkResources {
require.NotZero(t, nr.AccountSeqID, "fixture network_resource must have seq>0")
resourceSeqs[nr.ID] = nr.AccountSeqID
}
for _, nr := range account.NetworkRouters {
require.NotZero(t, nr.AccountSeqID, "fixture network_router must have seq>0")
routerSeqs[nr.ID] = nr.AccountSeqID
}
for _, n := range account.Networks {
require.NotZero(t, n.AccountSeqID, "fixture network must have seq>0 after backfill")
networkSeqs[n.ID] = n.AccountSeqID
}
require.NoError(t, store.SaveAccount(ctx, account))
after, err := store.GetAccount(ctx, accountID)
require.NoError(t, err)
for _, g := range after.Groups {
require.Equal(t, groupSeqs[g.ID], g.AccountSeqID, "group %s seq must be preserved on re-save", g.ID)
}
for _, p := range after.Policies {
require.Equal(t, policySeqs[p.ID], p.AccountSeqID, "policy %s seq must be preserved", p.ID)
}
for _, r := range after.Routes {
require.Equal(t, routeSeqs[r.ID], r.AccountSeqID, "route %s seq must be preserved (slice-of-value addressability)", r.ID)
}
for _, n := range after.NameServerGroups {
require.Equal(t, nsgSeqs[n.ID], n.AccountSeqID, "name_server_group %s seq must be preserved (slice-of-value addressability)", n.ID)
}
for _, nr := range after.NetworkResources {
require.Equal(t, resourceSeqs[nr.ID], nr.AccountSeqID, "network_resource %s seq must be preserved", nr.ID)
}
for _, nr := range after.NetworkRouters {
require.Equal(t, routerSeqs[nr.ID], nr.AccountSeqID, "network_router %s seq must be preserved", nr.ID)
}
for _, n := range after.Networks {
require.Equal(t, networkSeqs[n.ID], n.AccountSeqID, "network %s seq must be preserved", n.ID)
}
}
func TestSaveAccount_AllocatesSeqIDsForAllEntityTypes(t *testing.T) {
store, cleanup, err := NewTestStoreFromSQL(context.Background(), "", t.TempDir())
t.Cleanup(cleanup)
require.NoError(t, err)
ctx := context.Background()
const accountID = "save-account-all-entities"
addr, err := netip.ParseAddr("8.8.8.8")
require.NoError(t, err)
account := &types.Account{
Id: accountID,
CreatedBy: "user1",
Domain: "example.test",
Settings: &types.Settings{},
Network: &types.Network{Identifier: "net-test"},
Users: map[string]*types.User{
"user1": {Id: "user1", AccountID: accountID, Role: types.UserRoleOwner},
},
Groups: map[string]*types.Group{
"g1": {ID: "g1", AccountID: accountID, Name: "g1", Issued: types.GroupIssuedAPI},
},
Policies: []*types.Policy{
{ID: "p1", AccountID: accountID, Name: "p1", Enabled: true,
Rules: []*types.PolicyRule{{ID: "r1", PolicyID: "p1", Enabled: true}}},
},
Routes: map[route.ID]*route.Route{
"rt1": {ID: "rt1", AccountID: accountID, NetID: "net1", Peer: "peer1"},
},
NameServerGroups: map[string]*nbdns.NameServerGroup{
"nsg1": {ID: "nsg1", AccountID: accountID, Name: "nsg1", Enabled: true,
NameServers: []nbdns.NameServer{{IP: addr, NSType: nbdns.UDPNameServerType, Port: 53}}},
},
NetworkResources: []*resourceTypes.NetworkResource{
{ID: "nr1", AccountID: accountID, NetworkID: "net1", Name: "res1", Enabled: true},
},
NetworkRouters: []*routerTypes.NetworkRouter{
{ID: "nrt1", AccountID: accountID, NetworkID: "net1", Peer: "peer1", Enabled: true},
},
Networks: []*networkTypes.Network{
{ID: "n1", AccountID: accountID, Name: "n1"},
},
PostureChecks: []*posture.Checks{
{ID: "pc1", AccountID: accountID, Name: "pc1",
Checks: posture.ChecksDefinition{
NBVersionCheck: &posture.NBVersionCheck{MinVersion: "0.26.0"},
}},
},
}
require.NoError(t, store.SaveAccount(ctx, account))
after, err := store.GetAccount(ctx, accountID)
require.NoError(t, err)
require.Len(t, after.Groups, 1)
require.Len(t, after.Policies, 1)
require.Len(t, after.Routes, 1)
require.Len(t, after.NameServerGroups, 1)
require.Len(t, after.NetworkResources, 1)
require.Len(t, after.NetworkRouters, 1)
require.Len(t, after.Networks, 1)
require.Len(t, after.PostureChecks, 1)
for _, g := range after.Groups {
require.NotZero(t, g.AccountSeqID, "group seq must be allocated")
}
for _, p := range after.Policies {
require.NotZero(t, p.AccountSeqID, "policy seq must be allocated")
}
for _, r := range after.Routes {
require.NotZero(t, r.AccountSeqID, "route seq must be allocated (slice-of-value addressability)")
}
for _, n := range after.NameServerGroups {
require.NotZero(t, n.AccountSeqID, "name_server_group seq must be allocated (slice-of-value addressability)")
}
for _, nr := range after.NetworkResources {
require.NotZero(t, nr.AccountSeqID, "network_resource seq must be allocated")
}
for _, nr := range after.NetworkRouters {
require.NotZero(t, nr.AccountSeqID, "network_router seq must be allocated")
}
for _, n := range after.Networks {
require.NotZero(t, n.AccountSeqID, "network seq must be allocated")
}
for _, pc := range after.PostureChecks {
require.NotZero(t, pc.AccountSeqID, "posture_check seq must be allocated")
}
require.NoError(t, store.SaveAccount(ctx, after))
final, err := store.GetAccount(ctx, accountID)
require.NoError(t, err)
for _, r := range final.Routes {
require.Equal(t, after.Routes[r.ID].AccountSeqID, r.AccountSeqID, "route seq preserved on re-save")
}
for _, n := range final.NameServerGroups {
require.Equal(t, after.NameServerGroups[n.ID].AccountSeqID, n.AccountSeqID, "name_server_group seq preserved on re-save")
}
afterByID := map[string]uint32{}
for _, n := range after.Networks {
afterByID[n.ID] = n.AccountSeqID
}
for _, n := range final.Networks {
require.Equal(t, afterByID[n.ID], n.AccountSeqID, "network seq preserved on re-save")
}
afterPCByID := map[string]uint32{}
for _, pc := range after.PostureChecks {
afterPCByID[pc.ID] = pc.AccountSeqID
}
for _, pc := range final.PostureChecks {
require.Equal(t, afterPCByID[pc.ID], pc.AccountSeqID, "posture_check seq preserved on re-save")
}
}
func TestAllocateAccountSeqID_ConcurrentSameAccountEntity(t *testing.T) {
store, cleanup, err := NewTestStoreFromSQL(context.Background(), "", t.TempDir())
t.Cleanup(cleanup)
require.NoError(t, err)
ctx := context.Background()
const accountID = "concurrent-test"
const entity = types.AccountSeqEntityPolicy
const goroutines = 32
type result struct {
seq uint32
err error
}
results := make(chan result, goroutines)
start := make(chan struct{})
for i := 0; i < goroutines; i++ {
go func() {
<-start
var allocated uint32
err := store.ExecuteInTransaction(ctx, func(tx Store) error {
seq, err := tx.AllocateAccountSeqID(ctx, accountID, entity)
allocated = seq
return err
})
results <- result{seq: allocated, err: err}
}()
}
close(start)
seen := make(map[uint32]int, goroutines)
for i := 0; i < goroutines; i++ {
r := <-results
require.NoError(t, r.err, "concurrent allocate must not fail")
require.NotZero(t, r.seq, "allocated seq must be non-zero")
seen[r.seq]++
}
require.Lenf(t, seen, goroutines, "every concurrent allocation must yield a unique id; got duplicates in %v", seen)
for i := uint32(1); i <= goroutines; i++ {
require.Equalf(t, 1, seen[i], "id %d must appear exactly once across concurrent allocations", i)
}
}
func TestStoreCreateGroups_AllocatedSeqIDIsNotClobbered(t *testing.T) {
store, cleanup, err := NewTestStoreFromSQL(context.Background(), "../testdata/store.sql", t.TempDir())
t.Cleanup(cleanup)
require.NoError(t, err)
ctx := context.Background()
const accountID = "bf1c8084-ba50-4ce7-9439-34653001fc3b"
groups := []*types.Group{
{ID: "seq-test-g1", AccountID: accountID, Name: "g1", Issued: "jwt", AccountSeqID: 7777},
{ID: "seq-test-g2", AccountID: accountID, Name: "g2", Issued: "jwt", AccountSeqID: 7778},
}
require.NoError(t, store.CreateGroups(ctx, accountID, groups))
for _, want := range groups {
got, err := store.GetGroupByID(ctx, LockingStrengthNone, accountID, want.ID)
require.NoError(t, err)
require.Equal(t, want.AccountSeqID, got.AccountSeqID, "seq id from caller must be persisted on insert")
}
groups[0].Name = "g1-renamed"
groups[0].AccountSeqID = 0
require.NoError(t, store.CreateGroups(ctx, accountID, groups[:1]))
got, err := store.GetGroupByID(ctx, LockingStrengthNone, accountID, "seq-test-g1")
require.NoError(t, err)
require.Equal(t, "g1-renamed", got.Name, "upsert path still updates other columns")
require.Equal(t, uint32(7777), got.AccountSeqID, "upsert path must NOT overwrite account_seq_id")
}
func TestPolicyCreate_AllocatesSeqID(t *testing.T) {
store, cleanup, err := NewTestStoreFromSQL(context.Background(), "../testdata/store.sql", t.TempDir())
t.Cleanup(cleanup)
require.NoError(t, err)
ctx := context.Background()
const accountID = "bf1c8084-ba50-4ce7-9439-34653001fc3b"
existing, err := store.GetAccountPolicies(ctx, LockingStrengthNone, accountID)
require.NoError(t, err)
maxSeq := uint32(0)
for _, p := range existing {
if p.AccountSeqID > maxSeq {
maxSeq = p.AccountSeqID
}
}
require.NoError(t, store.ExecuteInTransaction(ctx, func(tx Store) error {
seq, err := tx.AllocateAccountSeqID(ctx, accountID, types.AccountSeqEntityPolicy)
if err != nil {
return err
}
require.Equal(t, maxSeq+1, seq, "next id should be max+1 after backfill")
newPolicy := &types.Policy{
ID: "bench-new-policy",
AccountID: accountID,
AccountSeqID: seq,
Enabled: true,
Rules: []*types.PolicyRule{{
ID: "bench-new-policy-rule",
PolicyID: "bench-new-policy",
Enabled: true,
Action: types.PolicyTrafficActionAccept,
Sources: []string{"groupA"},
Destinations: []string{"groupC"},
Bidirectional: true,
}},
}
return tx.CreatePolicy(ctx, newPolicy)
}))
created, err := store.GetPolicyByID(ctx, LockingStrengthNone, accountID, "bench-new-policy")
require.NoError(t, err)
require.Equal(t, maxSeq+1, created.AccountSeqID)
}
management/server/store/sql_store.go
+7 -268
View File
@@ -142,7 +142,6 @@ func NewSqlStore(ctx context.Context, db *gorm.DB, storeEngine types.Engine, met
&agentNetworkTypes.Consumption{}, &agentNetworkTypes.AccountBudgetRule{},
&agentNetworkTypes.AgentNetworkAccessLog{}, &agentNetworkTypes.AgentNetworkAccessLogGroup{},
&agentNetworkTypes.AgentNetworkUsage{}, &agentNetworkTypes.AgentNetworkUsageGroup{},
&types.AccountSeqCounter{},
)
if err != nil {
return nil, fmt.Errorf("auto migratePreAuto: %w", err)
@@ -314,10 +313,6 @@ func (s *SqlStore) SaveAccount(ctx context.Context, account *types.Account) erro
return result.Error
}
if err := s.assignAccountSeqIDs(ctx, tx, account); err != nil {
return fmt.Errorf("assign seq ids: %w", err)
}
result = tx.
Session(&gorm.Session{FullSaveAssociations: true}).
Clauses(clause.OnConflict{UpdateAll: true}).
@@ -2060,7 +2055,7 @@ func (s *SqlStore) getGroups(ctx context.Context, accountID string) ([]*types.Gr
var resources []byte
var refID sql.NullInt64
var refType sql.NullString
err := row.Scan(&g.ID, &g.AccountID, &g.AccountSeqID, &g.Name, &g.Issued, &resources, &refID, &refType)
err := row.Scan(&g.ID, &g.AccountID, &g.PublicID, &g.Name, &g.Issued, &resources, &refID, &refType)
if err == nil {
if refID.Valid {
g.IntegrationReference.ID = int(refID.Int64)
@@ -2094,7 +2089,7 @@ func (s *SqlStore) getPolicies(ctx context.Context, accountID string) ([]*types.
var p types.Policy
var checks []byte
var enabled sql.NullBool
err := row.Scan(&p.ID, &p.AccountID, &p.AccountSeqID, &p.Name, &p.Description, &enabled, &checks)
err := row.Scan(&p.ID, &p.AccountID, &p.PublicID, &p.Name, &p.Description, &enabled, &checks)
if err == nil {
if enabled.Valid {
p.Enabled = enabled.Bool
@@ -2122,7 +2117,7 @@ func (s *SqlStore) getRoutes(ctx context.Context, accountID string) ([]route.Rou
var network, domains, peerGroups, groups, accessGroups []byte
var keepRoute, masquerade, enabled, skipAutoApply sql.NullBool
var metric sql.NullInt64
err := row.Scan(&r.ID, &r.AccountID, &r.AccountSeqID, &network, &domains, &keepRoute, &r.NetID, &r.Description, &r.Peer, &peerGroups, &r.NetworkType, &masquerade, &metric, &enabled, &groups, &accessGroups, &skipAutoApply)
err := row.Scan(&r.ID, &r.AccountID, &r.PublicID, &network, &domains, &keepRoute, &r.NetID, &r.Description, &r.Peer, &peerGroups, &r.NetworkType, &masquerade, &metric, &enabled, &groups, &accessGroups, &skipAutoApply)
if err == nil {
if keepRoute.Valid {
r.KeepRoute = keepRoute.Bool
@@ -2173,7 +2168,7 @@ func (s *SqlStore) getNameServerGroups(ctx context.Context, accountID string) ([
var n nbdns.NameServerGroup
var ns, groups, domains []byte
var primary, enabled, searchDomainsEnabled sql.NullBool
err := row.Scan(&n.ID, &n.AccountID, &n.AccountSeqID, &n.Name, &n.Description, &ns, &groups, &primary, &domains, &enabled, &searchDomainsEnabled)
err := row.Scan(&n.ID, &n.AccountID, &n.PublicID, &n.Name, &n.Description, &ns, &groups, &primary, &domains, &enabled, &searchDomainsEnabled)
if err == nil {
if primary.Valid {
n.Primary = primary.Bool
@@ -2217,7 +2212,7 @@ func (s *SqlStore) getPostureChecks(ctx context.Context, accountID string) ([]*p
checks, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (*posture.Checks, error) {
var c posture.Checks
var checksDef []byte
err := row.Scan(&c.ID, &c.AccountID, &c.AccountSeqID, &c.Name, &c.Description, &checksDef)
err := row.Scan(&c.ID, &c.AccountID, &c.PublicID, &c.Name, &c.Description, &checksDef)
if err == nil && checksDef != nil {
_ = json.Unmarshal(checksDef, &c.Checks)
}
@@ -2424,7 +2419,7 @@ func (s *SqlStore) getNetworkRouters(ctx context.Context, accountID string) ([]*
var peerGroups []byte
var masquerade, enabled sql.NullBool
var metric sql.NullInt64
err := row.Scan(&r.ID, &r.NetworkID, &r.AccountID, &r.AccountSeqID, &r.Peer, &peerGroups, &masquerade, &metric, &enabled)
err := row.Scan(&r.ID, &r.NetworkID, &r.AccountID, &r.PublicID, &r.Peer, &peerGroups, &masquerade, &metric, &enabled)
if err == nil {
if masquerade.Valid {
r.Masquerade = masquerade.Bool
@@ -2461,7 +2456,7 @@ func (s *SqlStore) getNetworkResources(ctx context.Context, accountID string) ([
var r resourceTypes.NetworkResource
var prefix []byte
var enabled sql.NullBool
err := row.Scan(&r.ID, &r.NetworkID, &r.AccountID, &r.AccountSeqID, &r.Name, &r.Description, &r.Type, &r.Domain, &prefix, &enabled)
err := row.Scan(&r.ID, &r.NetworkID, &r.AccountID, &r.PublicID, &r.Name, &r.Description, &r.Type, &r.Domain, &prefix, &enabled)
if err == nil {
if enabled.Valid {
r.Enabled = enabled.Bool
@@ -3634,262 +3629,6 @@ func (s *SqlStore) withTx(tx *gorm.DB) Store {
}
}
// AllocateAccountSeqID returns the next per-account integer id for the given
// component kind. Must be called inside ExecuteInTransaction so the increment
// is serialized with the component insert.
func (s *SqlStore) AllocateAccountSeqID(ctx context.Context, accountID string, entity types.AccountSeqEntity) (int32, error) {
return allocateAccountSeqID(ctx, s.db, s.storeEngine, accountID, entity)
}
func allocateAccountSeqID(_ context.Context, db *gorm.DB, engine types.Engine, accountID string, entity types.AccountSeqEntity) (int32, error) {
switch engine {
case types.PostgresStoreEngine, types.SqliteStoreEngine:
return allocateAccountSeqIDReturning(db, accountID, entity)
case types.MysqlStoreEngine:
return allocateAccountSeqIDMysql(db, accountID, entity)
default:
return 0, fmt.Errorf("unsupported store engine for account_seq allocator: %v", engine)
}
}
// allocateAccountSeqIDReturning runs a single atomic INSERT ... ON CONFLICT
// DO UPDATE ... RETURNING that gives us the allocated id without a separate
// SELECT FOR UPDATE. Two concurrent allocations for the same (account, entity)
// produce two distinct ids: one wins the INSERT, the other wins the UPDATE
// branch and returns next_id+1.
func allocateAccountSeqIDReturning(db *gorm.DB, accountID string, entity types.AccountSeqEntity) (int32, error) {
const sqlStr = `
INSERT INTO account_seq_counters (account_id, entity, next_id)
VALUES (?, ?, 2)
ON CONFLICT (account_id, entity) DO UPDATE
SET next_id = account_seq_counters.next_id + 1
RETURNING (next_id - 1)
`
var allocated int32
if err := db.Raw(sqlStr, accountID, string(entity)).Scan(&allocated).Error; err != nil {
return 0, fmt.Errorf("upsert account seq counter: %w", err)
}
if allocated == 0 {
return 0, fmt.Errorf("upsert account seq counter returned 0")
}
return allocated, nil
}
// allocateAccountSeqIDMysql is the MySQL equivalent of allocateAccountSeqIDReturning.
// MySQL has no RETURNING on ON DUPLICATE KEY UPDATE, so we use the LAST_INSERT_ID
// trick: passing an expression to LAST_INSERT_ID(expr) both sets the session value
// and returns it from the INSERT. The INSERT's value uses LAST_INSERT_ID(2) so the
// no-conflict path also surfaces the new next_id, keeping the read-back uniform.
// LAST_INSERT_ID is per-connection; GORM transactions pin a single connection,
// so the follow-up SELECT sees the same value.
func allocateAccountSeqIDMysql(db *gorm.DB, accountID string, entity types.AccountSeqEntity) (int32, error) {
const upsertSQL = `
INSERT INTO account_seq_counters (account_id, entity, next_id)
VALUES (?, ?, LAST_INSERT_ID(2))
ON DUPLICATE KEY UPDATE next_id = LAST_INSERT_ID(next_id + 1)
`
if err := db.Exec(upsertSQL, accountID, string(entity)).Error; err != nil {
return 0, fmt.Errorf("upsert account seq counter: %w", err)
}
var newNext uint64
if err := db.Raw("SELECT LAST_INSERT_ID()").Scan(&newNext).Error; err != nil {
return 0, fmt.Errorf("get last insert id: %w", err)
}
if newNext == 0 {
return 0, fmt.Errorf("LAST_INSERT_ID returned 0; account_seq_counters misconfigured")
}
return int32(newNext - 1), nil
}
// assignAccountSeqIDs allocates a per-account integer id for any component on
// the in-memory account whose AccountSeqID is zero. Called from SaveAccount so
// the canonical "save the whole account" path produces the same persisted seq
// ids that the manager-level Create paths produce. Update flows that go
// through SaveAccount preserve existing non-zero values; for those, the
// per-entity counter is bumped so subsequent AllocateAccountSeqID calls don't
// hand out a colliding id.
func (s *SqlStore) assignAccountSeqIDs(ctx context.Context, tx *gorm.DB, account *types.Account) error {
maxByEntity := make(map[types.AccountSeqEntity]int32, 8)
bump := func(entity types.AccountSeqEntity, seq int32) {
if seq > maxByEntity[entity] {
maxByEntity[entity] = seq
}
}
for i := range account.GroupsG {
g := account.GroupsG[i]
if g == nil {
continue
}
if g.AccountSeqID != 0 {
bump(types.AccountSeqEntityGroup, g.AccountSeqID)
continue
}
seq, err := allocateAccountSeqID(ctx, tx, s.storeEngine, account.Id, types.AccountSeqEntityGroup)
if err != nil {
return err
}
g.AccountSeqID = seq
// Defensive: generateAccountSQLTypes currently aliases the same
// *Group pointer into GroupsG and Groups[id] (so this is a no-op
// today), but mirror the seq anyway so any future divergence in
// how the two collections are populated doesn't silently leave
// the canonical map view stale.
if original, ok := account.Groups[g.ID]; ok && original != nil && original != g {
original.AccountSeqID = seq
}
}
for _, p := range account.Policies {
if p == nil {
continue
}
if p.AccountSeqID != 0 {
bump(types.AccountSeqEntityPolicy, p.AccountSeqID)
continue
}
seq, err := allocateAccountSeqID(ctx, tx, s.storeEngine, account.Id, types.AccountSeqEntityPolicy)
if err != nil {
return err
}
p.AccountSeqID = seq
}
for i := range account.RoutesG {
r := &account.RoutesG[i]
if r.AccountSeqID != 0 {
bump(types.AccountSeqEntityRoute, r.AccountSeqID)
continue
}
seq, err := allocateAccountSeqID(ctx, tx, s.storeEngine, account.Id, types.AccountSeqEntityRoute)
if err != nil {
return err
}
r.AccountSeqID = seq
// Mirror the new seq onto the canonical map view so callers that
// hold the same in-memory account post-Save read a consistent
// AccountSeqID — without this, components/encoder code would see
// 0 for routes saved this transaction until the account is reloaded.
if original, ok := account.Routes[r.ID]; ok && original != nil {
original.AccountSeqID = seq
}
}
for i := range account.NameServerGroupsG {
ng := &account.NameServerGroupsG[i]
if ng.AccountSeqID != 0 {
bump(types.AccountSeqEntityNameserverGroup, ng.AccountSeqID)
continue
}
seq, err := allocateAccountSeqID(ctx, tx, s.storeEngine, account.Id, types.AccountSeqEntityNameserverGroup)
if err != nil {
return err
}
ng.AccountSeqID = seq
if original, ok := account.NameServerGroups[ng.ID]; ok && original != nil {
original.AccountSeqID = seq
}
}
for _, nr := range account.NetworkResources {
if nr == nil {
continue
}
if nr.AccountSeqID != 0 {
bump(types.AccountSeqEntityNetworkResource, nr.AccountSeqID)
continue
}
seq, err := allocateAccountSeqID(ctx, tx, s.storeEngine, account.Id, types.AccountSeqEntityNetworkResource)
if err != nil {
return err
}
nr.AccountSeqID = seq
}
for _, nr := range account.NetworkRouters {
if nr == nil {
continue
}
if nr.AccountSeqID != 0 {
bump(types.AccountSeqEntityNetworkRouter, nr.AccountSeqID)
continue
}
seq, err := allocateAccountSeqID(ctx, tx, s.storeEngine, account.Id, types.AccountSeqEntityNetworkRouter)
if err != nil {
return err
}
nr.AccountSeqID = seq
}
for _, n := range account.Networks {
if n == nil {
continue
}
if n.AccountSeqID != 0 {
bump(types.AccountSeqEntityNetwork, n.AccountSeqID)
continue
}
seq, err := allocateAccountSeqID(ctx, tx, s.storeEngine, account.Id, types.AccountSeqEntityNetwork)
if err != nil {
return err
}
n.AccountSeqID = seq
}
for _, pc := range account.PostureChecks {
if pc == nil {
continue
}
if pc.AccountSeqID != 0 {
bump(types.AccountSeqEntityPostureCheck, pc.AccountSeqID)
continue
}
seq, err := allocateAccountSeqID(ctx, tx, s.storeEngine, account.Id, types.AccountSeqEntityPostureCheck)
if err != nil {
return err
}
pc.AccountSeqID = seq
}
for entity, maxSeq := range maxByEntity {
if err := ensureAccountSeqCounter(tx, s.storeEngine, account.Id, entity, maxSeq+1); err != nil {
return fmt.Errorf("seed counter for %s: %w", entity, err)
}
}
return nil
}
// ensureAccountSeqCounter raises the per-account counter for entity to at
// least target. Used when SaveAccount persists components that already carry
// AccountSeqIDs (e.g. test bulk-load from sqlite to postgres, or migrations
// running before component data lands) so that the next AllocateAccountSeqID
// call returns a fresh id beyond what was just written.
func ensureAccountSeqCounter(db *gorm.DB, engine types.Engine, accountID string, entity types.AccountSeqEntity, target int32) error {
switch engine {
case types.PostgresStoreEngine, types.SqliteStoreEngine:
const sqlStr = `
INSERT INTO account_seq_counters (account_id, entity, next_id)
VALUES (?, ?, ?)
ON CONFLICT (account_id, entity) DO UPDATE
SET next_id = GREATEST(account_seq_counters.next_id, EXCLUDED.next_id)
`
// sqlite's UPSERT understands max() but the migration uses GREATEST
// for postgres and max() for sqlite. We collapse to dialect-specific
// statements only when needed.
if engine == types.SqliteStoreEngine {
const sqliteSQL = `
INSERT INTO account_seq_counters (account_id, entity, next_id)
VALUES (?, ?, ?)
ON CONFLICT (account_id, entity) DO UPDATE
SET next_id = max(account_seq_counters.next_id, excluded.next_id)
`
return db.Exec(sqliteSQL, accountID, string(entity), target).Error
}
return db.Exec(sqlStr, accountID, string(entity), target).Error
case types.MysqlStoreEngine:
const sqlStr = `
INSERT INTO account_seq_counters (account_id, entity, next_id)
VALUES (?, ?, ?)
ON DUPLICATE KEY UPDATE next_id = GREATEST(next_id, VALUES(next_id))
`
return db.Exec(sqlStr, accountID, string(entity), target).Error
default:
return fmt.Errorf("unsupported store engine for account_seq counter: %v", engine)
}
}
// transaction wraps a GORM transaction with MySQL-specific FK checks handling
// Use this instead of db.Transaction() directly to avoid deadlocks on MySQL/Aurora
func (s *SqlStore) transaction(fn func(*gorm.DB) error) error {
management/server/store/store.go
-29
View File
@@ -223,11 +223,6 @@ type Store interface {
GetStoreEngine() types.Engine
ExecuteInTransaction(ctx context.Context, f func(store Store) error) error
// AllocateAccountSeqID returns the next per-account integer id for the given
// component kind. Must run inside a transaction so the increment is serialized
// with the component insert.
AllocateAccountSeqID(ctx context.Context, accountID string, entity types.AccountSeqEntity) (int32, error)
GetAccountNetworks(ctx context.Context, lockStrength LockingStrength, accountID string) ([]*networkTypes.Network, error)
GetNetworkByID(ctx context.Context, lockStrength LockingStrength, accountID, networkID string) (*networkTypes.Network, error)
SaveNetwork(ctx context.Context, network *networkTypes.Network) error
@@ -629,30 +624,6 @@ func getMigrationsPostAuto(ctx context.Context) []migrationFunc {
func(db *gorm.DB) error {
return migration.DropIndex[proxy.Proxy](ctx, db, "idx_proxy_account_id_unique")
},
func(db *gorm.DB) error {
return migration.BackfillAccountSeqIDs[types.Policy](ctx, db, types.AccountSeqEntityPolicy, "id")
},
func(db *gorm.DB) error {
return migration.BackfillAccountSeqIDs[types.Group](ctx, db, types.AccountSeqEntityGroup, "id")
},
func(db *gorm.DB) error {
return migration.BackfillAccountSeqIDs[route.Route](ctx, db, types.AccountSeqEntityRoute, "id")
},
func(db *gorm.DB) error {
return migration.BackfillAccountSeqIDs[resourceTypes.NetworkResource](ctx, db, types.AccountSeqEntityNetworkResource, "id")
},
func(db *gorm.DB) error {
return migration.BackfillAccountSeqIDs[routerTypes.NetworkRouter](ctx, db, types.AccountSeqEntityNetworkRouter, "id")
},
func(db *gorm.DB) error {
return migration.BackfillAccountSeqIDs[dns.NameServerGroup](ctx, db, types.AccountSeqEntityNameserverGroup, "id")
},
func(db *gorm.DB) error {
return migration.BackfillAccountSeqIDs[networkTypes.Network](ctx, db, types.AccountSeqEntityNetwork, "id")
},
func(db *gorm.DB) error {
return migration.BackfillAccountSeqIDs[posture.Checks](ctx, db, types.AccountSeqEntityPostureCheck, "id")
},
}
}
management/server/store/store_mock.go
-15
View File
@@ -138,21 +138,6 @@ func (mr *MockStoreMockRecorder) AddResourceToGroup(ctx, accountId, groupID, res
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddResourceToGroup", reflect.TypeOf((*MockStore)(nil).AddResourceToGroup), ctx, accountId, groupID, resource)
}
// AllocateAccountSeqID mocks base method.
func (m *MockStore) AllocateAccountSeqID(ctx context.Context, accountID string, entity types3.AccountSeqEntity) (int32, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "AllocateAccountSeqID", ctx, accountID, entity)
ret0, _ := ret[0].(int32)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// AllocateAccountSeqID indicates an expected call of AllocateAccountSeqID.
func (mr *MockStoreMockRecorder) AllocateAccountSeqID(ctx, accountID, entity interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocateAccountSeqID", reflect.TypeOf((*MockStore)(nil).AllocateAccountSeqID), ctx, accountID, entity)
}
// ApproveAccountPeers mocks base method.
func (m *MockStore) ApproveAccountPeers(ctx context.Context, accountID string) (int, error) {
m.ctrl.T.Helper()
management/server/types/account_components.go
+15 -15
View File
@@ -125,26 +125,26 @@ func (a *Account) GetPeerNetworkMapComponents(
}
components := &NetworkMapComponents{
PeerID: peerID,
Network: a.Network.Copy(),
NameServerGroups: make([]*nbdns.NameServerGroup, 0),
CustomZoneDomain: peersCustomZone.Domain,
ResourcePoliciesMap: make(map[string][]*Policy),
RoutersMap: make(map[string]map[string]*routerTypes.NetworkRouter),
NetworkResources: make([]*resourceTypes.NetworkResource, 0),
PostureFailedPeers: make(map[string]map[string]struct{}, len(a.PostureChecks)),
RouterPeers: make(map[string]*nbpeer.Peer),
NetworkXIDToSeq: make(map[string]int32, len(a.Networks)),
PostureCheckXIDToSeq: make(map[string]int32, len(a.PostureChecks)),
PeerID: peerID,
Network: a.Network.Copy(),
NameServerGroups: make([]*nbdns.NameServerGroup, 0),
CustomZoneDomain: peersCustomZone.Domain,
ResourcePoliciesMap: make(map[string][]*Policy),
RoutersMap: make(map[string]map[string]*routerTypes.NetworkRouter),
NetworkResources: make([]*resourceTypes.NetworkResource, 0),
PostureFailedPeers: make(map[string]map[string]struct{}, len(a.PostureChecks)),
RouterPeers: make(map[string]*nbpeer.Peer),
NetworkXIDToPublicID: make(map[string]string, len(a.Networks)),
PostureCheckXIDToPublicID: make(map[string]string, len(a.PostureChecks)),
}
for _, n := range a.Networks {
if n != nil && n.HasSeqID() {
components.NetworkXIDToSeq[n.ID] = n.AccountSeqID
if n != nil {
components.NetworkXIDToPublicID[n.ID] = n.PublicID
}
}
for _, pc := range a.PostureChecks {
if pc != nil && pc.HasSeqID() {
components.PostureCheckXIDToSeq[pc.ID] = pc.AccountSeqID
if pc != nil {
components.PostureCheckXIDToPublicID[pc.ID] = pc.PublicID
}
}
management/server/types/account_seq_counter.go
-29
View File
@@ -1,29 +0,0 @@
package types
// AccountSeqEntity identifies the kind of component that uses a per-account sequence.
type AccountSeqEntity string
const (
AccountSeqEntityPolicy AccountSeqEntity = "policy"
AccountSeqEntityGroup AccountSeqEntity = "group"
AccountSeqEntityRoute AccountSeqEntity = "route"
AccountSeqEntityNetworkResource AccountSeqEntity = "network_resource"
AccountSeqEntityNetworkRouter AccountSeqEntity = "network_router"
AccountSeqEntityNameserverGroup AccountSeqEntity = "nameserver_group"
AccountSeqEntityNetwork AccountSeqEntity = "network"
AccountSeqEntityPostureCheck AccountSeqEntity = "posture_check"
)
// AccountSeqCounter tracks the next per-account integer id for a given component
// kind. Reads/writes go through the store inside the same transaction as the
// component insert so two concurrent inserts cannot collide on the same id.
type AccountSeqCounter struct {
AccountID string `gorm:"primaryKey;size:255"`
Entity string `gorm:"primaryKey;size:32"`
NextID uint32 `gorm:"not null;default:1"`
}
// TableName overrides the GORM-derived table name.
func (AccountSeqCounter) TableName() string {
return "account_seq_counters"
}
management/server/types/networkmap_components_correctness_test.go
+12 -10
View File
@@ -9,6 +9,7 @@ import (
"testing"
"time"
"github.com/rs/xid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -88,13 +89,13 @@ func buildScalableTestAccount(numPeers, numGroups int, withDefaultPolicy bool) (
for i := start; i < end; i++ {
groupPeers = append(groupPeers, fmt.Sprintf("peer-%d", i))
}
groups[groupID] = &types.Group{ID: groupID, Name: fmt.Sprintf("Group %d", g), Peers: groupPeers}
groups[groupID] = &types.Group{ID: groupID, PublicID: xid.New().String(), Name: fmt.Sprintf("Group %d", g), Peers: groupPeers}
}
policies := make([]*types.Policy, 0, numGroups+2)
if withDefaultPolicy {
policies = append(policies, &types.Policy{
ID: "policy-all", Name: "Default-Allow", Enabled: true,
ID: "policy-all", PublicID: xid.New().String(), Name: "Default-Allow", Enabled: true,
Rules: []*types.PolicyRule{{
ID: "rule-all", Name: "Allow All", Enabled: true, Action: types.PolicyTrafficActionAccept,
Protocol: types.PolicyRuleProtocolALL, Bidirectional: true,
@@ -107,7 +108,7 @@ func buildScalableTestAccount(numPeers, numGroups int, withDefaultPolicy bool) (
groupID := fmt.Sprintf("group-%d", g)
dstGroup := fmt.Sprintf("group-%d", (g+1)%numGroups)
policies = append(policies, &types.Policy{
ID: fmt.Sprintf("policy-%d", g), Name: fmt.Sprintf("Policy %d", g), Enabled: true,
ID: fmt.Sprintf("policy-%d", g), PublicID: xid.New().String(), Name: fmt.Sprintf("Policy %d", g), Enabled: true,
Rules: []*types.PolicyRule{{
ID: fmt.Sprintf("rule-%d", g), Name: fmt.Sprintf("Rule %d", g), Enabled: true,
Action: types.PolicyTrafficActionAccept, Protocol: types.PolicyRuleProtocolTCP,
@@ -120,7 +121,7 @@ func buildScalableTestAccount(numPeers, numGroups int, withDefaultPolicy bool) (
if numGroups >= 2 {
policies = append(policies, &types.Policy{
ID: "policy-drop", Name: "Drop DB traffic", Enabled: true,
ID: "policy-drop", PublicID: xid.New().String(), Name: "Drop DB traffic", Enabled: true,
Rules: []*types.PolicyRule{{
ID: "rule-drop", Name: "Drop DB", Enabled: true, Action: types.PolicyTrafficActionDrop,
Protocol: types.PolicyRuleProtocolTCP, Ports: []string{"5432"}, Bidirectional: true,
@@ -144,6 +145,7 @@ func buildScalableTestAccount(numPeers, numGroups int, withDefaultPolicy bool) (
groupID := fmt.Sprintf("group-%d", r%numGroups)
routes[routeID] = &route.Route{
ID: routeID,
PublicID: xid.New().String(),
Network: netip.MustParsePrefix(fmt.Sprintf("10.%d.0.0/16", r)),
Peer: peers[routePeerID].Key,
PeerID: routePeerID,
@@ -178,18 +180,18 @@ func buildScalableTestAccount(numPeers, numGroups int, withDefaultPolicy bool) (
}
routerPeerID := fmt.Sprintf("peer-%d", routerPeerIdx)
networksList = append(networksList, &networkTypes.Network{ID: netID, Name: fmt.Sprintf("Network %d", nr), AccountID: "test-account"})
networksList = append(networksList, &networkTypes.Network{ID: netID, PublicID: xid.New().String(), Name: fmt.Sprintf("Network %d", nr), AccountID: "test-account"})
networkResources = append(networkResources, &resourceTypes.NetworkResource{
ID: resID, NetworkID: netID, AccountID: "test-account", Enabled: true,
ID: resID, PublicID: xid.New().String(), NetworkID: netID, AccountID: "test-account", Enabled: true,
Address: fmt.Sprintf("svc-%d.netbird.cloud", nr),
})
networkRouters = append(networkRouters, &routerTypes.NetworkRouter{
ID: fmt.Sprintf("router-%d", nr), NetworkID: netID, Peer: routerPeerID,
ID: fmt.Sprintf("router-%d", nr), PublicID: xid.New().String(), NetworkID: netID, Peer: routerPeerID,
Enabled: true, AccountID: "test-account",
})
policies = append(policies, &types.Policy{
ID: fmt.Sprintf("policy-res-%d", nr), Name: fmt.Sprintf("Resource Policy %d", nr), Enabled: true,
ID: fmt.Sprintf("policy-res-%d", nr), PublicID: xid.New().String(), Name: fmt.Sprintf("Resource Policy %d", nr), Enabled: true,
SourcePostureChecks: []string{"posture-check-ver"},
Rules: []*types.PolicyRule{{
ID: fmt.Sprintf("rule-res-%d", nr), Name: fmt.Sprintf("Allow Resource %d", nr), Enabled: true,
@@ -215,12 +217,12 @@ func buildScalableTestAccount(numPeers, numGroups int, withDefaultPolicy bool) (
DNSSettings: types.DNSSettings{DisabledManagementGroups: []string{}},
NameServerGroups: map[string]*nbdns.NameServerGroup{
"ns-group-main": {
ID: "ns-group-main", Name: "Main NS", Enabled: true, Groups: []string{"group-all"},
ID: "ns-group-main", PublicID: xid.New().String(), Name: "Main NS", Enabled: true, Groups: []string{"group-all"},
NameServers: []nbdns.NameServer{{IP: netip.MustParseAddr("8.8.8.8"), NSType: nbdns.UDPNameServerType, Port: 53}},
},
},
PostureChecks: []*posture.Checks{
{ID: "posture-check-ver", Name: "Check version", Checks: posture.ChecksDefinition{
{ID: "posture-check-ver", PublicID: xid.New().String(), Name: "Check version", Checks: posture.ChecksDefinition{
NBVersionCheck: &posture.NBVersionCheck{MinVersion: "0.26.0"},
}},
},
management/server/types/networkmap_wire_benchmark_test.go
+2 -19
View File
@@ -24,23 +24,6 @@ var wireBenchScales = []benchmarkScale{
{"5000peers_100groups", 5000, 100},
}
// populateAccountSeqIDs assigns deterministic AccountSeqIDs to every group and
// policy in the account so that the component encoder can reference them. The
// scalableTestAccount fixture builds entities by struct literal and skips this
// step, but production paths populate the IDs via the store layer.
func populateAccountSeqIDs(account *types.Account) {
var nextGroupSeq uint32 = 1
for _, g := range account.Groups {
g.AccountSeqID = nextGroupSeq
nextGroupSeq++
}
var nextPolicySeq uint32 = 1
for _, p := range account.Policies {
p.AccountSeqID = nextPolicySeq
nextPolicySeq++
}
}
// assignValidWgKeys overwrites every peer's Key with a valid base64-encoded
// 32-byte string. The default scalableTestAccount uses unparsable strings
// like "key-peer-0", which makes the components encoder emit a nil WgPubKey
@@ -64,7 +47,7 @@ func BenchmarkNetworkMapWireEncode(b *testing.B) {
for _, scale := range wireBenchScales {
account, validatedPeers := scalableTestAccount(scale.peers, scale.groups)
populateAccountSeqIDs(account)
// populateAccountSeqIDs(account)
assignValidWgKeys(account)
ctx := context.Background()
@@ -135,7 +118,7 @@ func BenchmarkNetworkMapWireSize(b *testing.B) {
for _, scale := range wireBenchScales {
account, validatedPeers := scalableTestAccount(scale.peers, scale.groups)
populateAccountSeqIDs(account)
// populateAccountSeqIDs(account)
assignValidWgKeys(account)
ctx := context.Background()
management/server/types/networkmap_wire_breakdown_test.go
-1
View File
@@ -30,7 +30,6 @@ func TestNetworkMapWireBreakdown(t *testing.T) {
const peerCount, groupCount = 5000, 100
account, validatedPeers := scalableTestAccount(peerCount, groupCount)
populateAccountSeqIDs(account)
assignValidWgKeys(account)
ctx := context.Background()