Merge branch 'main' into fix/pkce-flow-session-extend

# Conflicts:
#	client/ios/NetBirdSDK/login.go
#	client/server/server.go
#	shared/management/proto/management.pb.go
This commit is contained in:
Zoltán Papp
2026-10-05 15:37:54 +02:00
847 changed files with 67577 additions and 25637 deletions
+190
View File
@@ -0,0 +1,190 @@
package acl
import (
"net/netip"
"sync"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"github.com/netbirdio/netbird/client/firewall"
fwmgr "github.com/netbirdio/netbird/client/firewall/manager"
"github.com/netbirdio/netbird/client/iface"
"github.com/netbirdio/netbird/client/iface/wgaddr"
"github.com/netbirdio/netbird/client/internal/acl/mocks"
mgmProto "github.com/netbirdio/netbird/shared/management/proto"
)
// TestNetworkZeroPrefixIsRoute guards the route-vs-peer dispatch
// invariant: the backends classify a rule as a peer rule purely by the
// absence of a destination (neither prefix nor set). A default route
// (0.0.0.0/0 or ::/0) is a valid prefix and must therefore classify as
// a route, not collapse into the peer path.
func TestNetworkZeroPrefixIsRoute(t *testing.T) {
for _, p := range []string{"0.0.0.0/0", "::/0", "10.0.0.0/8"} {
n := fwmgr.Network{Prefix: netip.MustParsePrefix(p)}
assert.True(t, n.IsPrefix(), "%s must report IsPrefix", p)
assert.True(t, n.IsPrefix() || n.IsSet(), "%s must classify as a route", p)
}
// A zero-value Network is the only peer-rule shape.
var empty fwmgr.Network
assert.False(t, empty.IsPrefix(), "zero Network must not be a prefix")
assert.False(t, empty.IsSet(), "zero Network must not be a set")
}
// TestDetermineDestinationAlwaysRoute verifies determineDestination
// never yields an empty Network for a valid route rule: every branch
// (static prefix, default route, dynamic with/without domains, with and
// without a local resolver) produces a destination that classifies as a
// route. If this regresses, a route rule would be dispatched down the
// peer path, which matches on source only.
func TestDetermineDestinationAlwaysRoute(t *testing.T) {
v4 := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")}
v6 := []netip.Prefix{netip.MustParsePrefix("2001:db8::/48")}
cases := []struct {
name string
rule *mgmProto.RouteFirewallRule
resolver bool
sources []netip.Prefix
}{
{"static prefix", &mgmProto.RouteFirewallRule{Destination: "192.168.0.0/16"}, false, v4},
{"static default route", &mgmProto.RouteFirewallRule{Destination: "0.0.0.0/0"}, false, v4},
{"dynamic with domains + resolver", &mgmProto.RouteFirewallRule{IsDynamic: true, Domains: []string{"example.com"}}, true, v4},
{"dynamic no domains + resolver (v4)", &mgmProto.RouteFirewallRule{IsDynamic: true}, true, v4},
{"dynamic no domains + resolver (v6)", &mgmProto.RouteFirewallRule{IsDynamic: true}, true, v6},
{"dynamic + no local resolver (v4)", &mgmProto.RouteFirewallRule{IsDynamic: true}, false, v4},
{"dynamic + no local resolver (v6)", &mgmProto.RouteFirewallRule{IsDynamic: true}, false, v6},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
dest, err := determineDestination(tc.rule, tc.resolver, tc.sources)
require.NoError(t, err)
assert.True(t, dest.IsPrefix() || dest.IsSet(),
"destination must classify as a route, got empty Network")
})
}
}
// countingFirewall wraps a real firewall.Manager and counts filter-rule
// add/delete calls so a test can assert how many backing rules the acl
// manager actually creates and tears down.
type countingFirewall struct {
fwmgr.Manager
mu sync.Mutex
addCalls int
dels int
ruleIDs map[fwmgr.RuleID]struct{}
}
// distinctRules returns the number of distinct backing rules the
// backend produced. Because the backend dedups identical content,
// repeated AddFilterRule calls for the same rule resolve to one id.
func (f *countingFirewall) distinctRules() int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.ruleIDs)
}
func (f *countingFirewall) AddFilterRule(id []byte, sources []netip.Prefix, destination fwmgr.Network, proto fwmgr.Protocol, sPort, dPort *fwmgr.Port, action fwmgr.Action) (fwmgr.Rule, error) {
rule, err := f.Manager.AddFilterRule(id, sources, destination, proto, sPort, dPort, action)
if err == nil {
f.mu.Lock()
f.addCalls++
if f.ruleIDs == nil {
f.ruleIDs = make(map[fwmgr.RuleID]struct{})
}
if rule != nil {
f.ruleIDs[rule.ID()] = struct{}{}
}
f.mu.Unlock()
}
return rule, err
}
func (f *countingFirewall) DeleteFilterRule(r fwmgr.Rule) error {
err := f.Manager.DeleteFilterRule(r)
if err == nil {
f.mu.Lock()
f.dels++
delete(f.ruleIDs, r.ID())
f.mu.Unlock()
}
return err
}
func newCountingACL(t *testing.T) (*DefaultManager, *countingFirewall, func()) {
t.Helper()
t.Setenv("NB_WG_KERNEL_DISABLED", "true")
t.Setenv(firewall.EnvForceUserspaceFirewall, "true")
ctrl := gomock.NewController(t)
ifaceMock := mocks.NewMockIFaceMapper(ctrl)
ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes()
ifaceMock.EXPECT().SetFilter(gomock.Any())
network := netip.MustParsePrefix("172.0.0.1/32")
ifaceMock.EXPECT().Name().Return("lo").AnyTimes()
ifaceMock.EXPECT().Address().Return(wgaddr.Address{IP: network.Addr(), Network: network}).AnyTimes()
ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes()
realFW, err := firewall.NewFirewall(ifaceMock, nil, flowLogger, false, iface.DefaultMTU)
require.NoError(t, err)
fw := &countingFirewall{Manager: realFW}
cleanup := func() {
require.NoError(t, realFW.Close(nil))
ctrl.Finish()
}
return NewDefaultManager(fw), fw, cleanup
}
// TestDuplicateContentPoliciesShareOneRule verifies the dedup contract
// the backends rely on: two policies that authorize an identical flow
// (same selector and sources) collapse to a single backing firewall
// rule, and that rule survives until BOTH policies are gone. This is
// why the backend can dedup on add without refcounting on delete: the
// acl manager's pair key matches the backend's content key, so add and
// delete stay balanced per content key across full-state reapplies.
func TestDuplicateContentPoliciesShareOneRule(t *testing.T) {
acl, fw, cleanup := newCountingACL(t)
defer cleanup()
ruleA := &mgmProto.FirewallRule{
PolicyID: []byte("policy-A"),
PeerIP: "10.0.0.1", //nolint:staticcheck
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_TCP,
Port: "443",
}
ruleB := &mgmProto.FirewallRule{
PolicyID: []byte("policy-B"),
PeerIP: "10.0.0.1", //nolint:staticcheck
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_TCP,
Port: "443",
}
// Both policies present: identical content collapses to one rule.
acl.ApplyFiltering(&mgmProto.NetworkMap{FirewallRules: []*mgmProto.FirewallRule{ruleA, ruleB}, FirewallRulesIsEmpty: false}, false)
assert.Equal(t, 1, fw.distinctRules(), "identical-content policies must produce one backing rule")
assert.Equal(t, 1, len(acl.peerRulesPairs), "one content key, one pair")
// Drop policy A only: the shared rule is still authorized by B, so
// nothing is deleted.
acl.ApplyFiltering(&mgmProto.NetworkMap{FirewallRules: []*mgmProto.FirewallRule{ruleB}, FirewallRulesIsEmpty: false}, false)
assert.Equal(t, 1, fw.distinctRules(), "no new backing rule on reapply")
assert.Equal(t, 0, fw.dels, "rule must survive while any policy still authorizes it")
assert.Equal(t, 1, len(acl.peerRulesPairs))
// Drop policy B too: now the content key has no authorizer and the
// single backing rule is removed exactly once.
acl.ApplyFiltering(&mgmProto.NetworkMap{FirewallRules: nil, FirewallRulesIsEmpty: true}, false)
assert.Equal(t, 1, fw.dels, "rule removed once when last policy is gone")
assert.Equal(t, 0, len(acl.peerRulesPairs))
}
+318
View File
@@ -0,0 +1,318 @@
package acl
import (
"errors"
"net/netip"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"github.com/netbirdio/netbird/client/firewall"
fwmgr "github.com/netbirdio/netbird/client/firewall/manager"
"github.com/netbirdio/netbird/client/iface"
"github.com/netbirdio/netbird/client/iface/wgaddr"
"github.com/netbirdio/netbird/client/internal/acl/mocks"
mgmProto "github.com/netbirdio/netbird/shared/management/proto"
"github.com/netbirdio/netbird/shared/netiputil"
)
// TestGroupPeerRulesPolicyIDSeparates verifies that two FirewallRules
// with identical selectors but different PolicyIDs do NOT get merged
// into one group, so each policy's sources merge under its own
// attribution id. (Identical-content groups may still dedup to one
// backing rule at the backend; see TestDuplicateContentPoliciesShareOneRule.)
func TestGroupPeerRulesPolicyIDSeparates(t *testing.T) {
rules := []*mgmProto.FirewallRule{
{
PolicyID: []byte("policy-A"),
PeerIP: "10.0.0.1",
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_TCP,
Port: "443",
},
{
PolicyID: []byte("policy-B"),
PeerIP: "10.0.0.1",
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_TCP,
Port: "443",
},
}
groups, denyErr, err := groupPeerRules(rules)
require.NoError(t, denyErr)
require.NoError(t, err)
require.Len(t, groups, 2, "rules with different PolicyIDs must produce separate groups")
}
// TestGroupPeerRulesFamilySeparates verifies that v4 and v6 rules
// belonging to the same policy don't merge.
func TestGroupPeerRulesFamilySeparates(t *testing.T) {
rules := []*mgmProto.FirewallRule{
{
PolicyID: []byte("policy-A"),
PeerIP: "10.0.0.1",
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_TCP,
Port: "443",
},
{
PolicyID: []byte("policy-A"),
PeerIP: "2001:db8::1",
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_TCP,
Port: "443",
},
}
groups, denyErr, err := groupPeerRules(rules)
require.NoError(t, denyErr)
require.NoError(t, err)
require.Len(t, groups, 2, "rules of different families must produce separate groups")
var sawV4, sawV6 bool
for _, g := range groups {
require.Len(t, g.sources, 1)
if g.sources[0].Addr().Is4() {
sawV4 = true
}
if g.sources[0].Addr().Is6() {
sawV6 = true
}
}
assert.True(t, sawV4 && sawV6)
}
// TestGroupPeerRulesSplitsMixedFamilySingleRule verifies that a single
// FirewallRule carrying both v4 and v6 source prefixes is split into one
// group per family. Each backend keys a rule to a single family, so a
// group whose sources span families would mismatch the other family's
// sources. mgmt normally emits one rule per family; this guards against
// a mixed-family rule slipping through.
func TestGroupPeerRulesSplitsMixedFamilySingleRule(t *testing.T) {
srcs := [][]byte{
netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.1")),
netiputil.EncodeAddr(netip.MustParseAddr("2001:db8::1")),
netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.2")),
netiputil.EncodeAddr(netip.MustParseAddr("2001:db8::2")),
}
rules := []*mgmProto.FirewallRule{
{
PolicyID: []byte("policy-A"),
SourcePrefixes: srcs,
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_TCP,
Port: "443",
},
}
groups, denyErr, err := groupPeerRules(rules)
require.NoError(t, denyErr)
require.NoError(t, err)
require.Len(t, groups, 2, "mixed-family sources in one rule must split into two groups")
for _, g := range groups {
require.Len(t, g.sources, 2)
v6 := prefixIsV6(g.sources[0])
for _, s := range g.sources {
assert.Equal(t, v6, prefixIsV6(s), "every source in a group must share one family")
}
}
}
// TestGroupPeerRulesMergesSameSelector verifies that rules sharing
// every distinguishing field (policy, family, direction, action,
// proto, port) collapse into a single multi-source group.
func TestGroupPeerRulesMergesSameSelector(t *testing.T) {
mk := func(peerIP string) *mgmProto.FirewallRule {
return &mgmProto.FirewallRule{
PolicyID: []byte("policy-A"),
PeerIP: peerIP, //nolint:staticcheck
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_TCP,
Port: "443",
}
}
rules := []*mgmProto.FirewallRule{mk("10.0.0.1"), mk("10.0.0.2"), mk("10.0.0.3")}
groups, denyErr, err := groupPeerRules(rules)
require.NoError(t, denyErr)
require.NoError(t, err)
require.Len(t, groups, 1)
require.Len(t, groups[0].sources, 3)
}
// TestGroupPeerRulesPortSeparates verifies that PortInfo is part of the
// selector key: rules differing only in port must not merge, and a
// single port must not merge with a range. A regression dropping the
// port from the key would collapse rules for different ports into one.
func TestGroupPeerRulesPortSeparates(t *testing.T) {
mkPort := func(peerIP string, port uint32) *mgmProto.FirewallRule {
return &mgmProto.FirewallRule{
PolicyID: []byte("policy-A"),
PeerIP: peerIP, //nolint:staticcheck
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_TCP,
PortInfo: &mgmProto.PortInfo{PortSelection: &mgmProto.PortInfo_Port{Port: port}},
}
}
groups, denyErr, err := groupPeerRules([]*mgmProto.FirewallRule{
mkPort("10.0.0.1", 80), mkPort("10.0.0.2", 80), mkPort("10.0.0.3", 443),
})
require.NoError(t, denyErr)
require.NoError(t, err)
require.Len(t, groups, 2, "rules on different ports must not merge")
rangeRule := &mgmProto.FirewallRule{
PolicyID: []byte("policy-A"),
PeerIP: "10.0.0.4", //nolint:staticcheck
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_TCP,
PortInfo: &mgmProto.PortInfo{PortSelection: &mgmProto.PortInfo_Range_{Range: &mgmProto.PortInfo_Range{Start: 80, End: 90}}},
}
groups, denyErr, err = groupPeerRules([]*mgmProto.FirewallRule{mkPort("10.0.0.1", 80), rangeRule})
require.NoError(t, denyErr)
require.NoError(t, err)
require.Len(t, groups, 2, "a single port and a range must not merge")
}
// TestGroupPeerRulesUsesSourcePrefixesWhenPresent verifies that the
// new sourcePrefixes wire field is consumed and produces a
// multi-source group in one shot (no client-side merging needed).
func TestGroupPeerRulesUsesSourcePrefixesWhenPresent(t *testing.T) {
srcs := [][]byte{
netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.1")),
netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.2")),
netiputil.EncodeAddr(netip.MustParseAddr("10.0.0.3")),
}
rules := []*mgmProto.FirewallRule{
{
PolicyID: []byte("policy-A"),
SourcePrefixes: srcs,
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_TCP,
Port: "443",
},
}
groups, denyErr, err := groupPeerRules(rules)
require.NoError(t, denyErr)
require.NoError(t, err)
require.Len(t, groups, 1)
require.Len(t, groups[0].sources, 3)
}
// TestGroupPeerRulesActionSeparates verifies the obvious: accept
// and drop rules with the same selector don't merge.
func TestGroupPeerRulesActionSeparates(t *testing.T) {
rules := []*mgmProto.FirewallRule{
{
PolicyID: []byte("policy-A"),
PeerIP: "10.0.0.1",
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_TCP,
Port: "443",
},
{
PolicyID: []byte("policy-A"),
PeerIP: "10.0.0.1",
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_DROP,
Protocol: mgmProto.RuleProtocol_TCP,
Port: "443",
},
}
groups, denyErr, err := groupPeerRules(rules)
require.NoError(t, denyErr)
require.NoError(t, err)
require.Len(t, groups, 2)
}
// failingDeleteFirewall wraps a real firewall.Manager and forces the
// next N DeleteFilterRule calls to fail. Used to verify that the acl
// manager retains rules whose deletion was rejected by the backend,
// so they get retried on the next ApplyFiltering pass instead of
// becoming orphans.
type failingDeleteFirewall struct {
fwmgr.Manager
failCount int
}
func (f *failingDeleteFirewall) DeleteFilterRule(r fwmgr.Rule) error {
if f.failCount > 0 {
f.failCount--
return errors.New("simulated delete failure")
}
return f.Manager.DeleteFilterRule(r)
}
// TestApplyFilteringRetainsRulesOnDeleteFailure verifies that a
// transient DeleteFilterRule error doesn't make the acl manager
// forget about a rule. The rule must remain in peerRulesPairs so the
// next ApplyFiltering pass attempts the delete again.
func TestApplyFilteringRetainsRulesOnDeleteFailure(t *testing.T) {
t.Setenv("NB_WG_KERNEL_DISABLED", "true")
t.Setenv(firewall.EnvForceUserspaceFirewall, "true")
ctrl := gomock.NewController(t)
defer ctrl.Finish()
ifaceMock := mocks.NewMockIFaceMapper(ctrl)
ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes()
ifaceMock.EXPECT().SetFilter(gomock.Any())
network := netip.MustParsePrefix("172.0.0.1/32")
ifaceMock.EXPECT().Name().Return("lo").AnyTimes()
ifaceMock.EXPECT().Address().Return(wgaddr.Address{IP: network.Addr(), Network: network}).AnyTimes()
ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes()
realFW, err := firewall.NewFirewall(ifaceMock, nil, flowLogger, false, iface.DefaultMTU)
require.NoError(t, err)
defer func() { require.NoError(t, realFW.Close(nil)) }()
fw := &failingDeleteFirewall{Manager: realFW}
acl := NewDefaultManager(fw)
// First pass: install a rule.
netmap1 := &mgmProto.NetworkMap{
FirewallRules: []*mgmProto.FirewallRule{
{
PolicyID: []byte("policy-A"),
PeerIP: "10.0.0.1",
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_DROP,
Protocol: mgmProto.RuleProtocol_TCP,
Port: "22",
},
},
FirewallRulesIsEmpty: false,
}
acl.ApplyFiltering(netmap1, false)
require.Equal(t, 1, len(acl.peerRulesPairs), "rule should be installed")
// Second pass: remove the rule from the map. The backend will
// fail the delete; the acl manager must retain the rule.
fw.failCount = 1
netmap2 := &mgmProto.NetworkMap{FirewallRules: nil, FirewallRulesIsEmpty: true}
acl.ApplyFiltering(netmap2, false)
require.Equal(t, 1, len(acl.peerRulesPairs),
"rule must be retained when DeleteFilterRule fails so it gets retried")
// Third pass: same map, backend no longer fails. The rule
// should now succeed in being removed.
acl.ApplyFiltering(netmap2, false)
require.Equal(t, 0, len(acl.peerRulesPairs), "retry should succeed")
}
+7 -6
View File
@@ -5,18 +5,18 @@ import (
"encoding/hex"
"fmt"
"net/netip"
"slices"
"strconv"
"github.com/netbirdio/netbird/client/firewall/manager"
)
type RuleID string
// RuleID aliases manager.RuleID so existing nbid.RuleID references
// keep working while the canonical type lives in the firewall package.
type RuleID = manager.RuleID
func (r RuleID) ID() string {
return string(r)
}
func GenerateRouteRuleKey(
// GenerateRuleID returns a deterministic content hash identifying a filter rule.
func GenerateRuleID(
sources []netip.Prefix,
destination manager.Network,
proto manager.Protocol,
@@ -24,6 +24,7 @@ func GenerateRouteRuleKey(
dPort *manager.Port,
action manager.Action,
) RuleID {
sources = slices.Clone(sources)
manager.SortPrefixes(sources)
h := sha256.New()
@@ -0,0 +1,75 @@
package acl
import (
"net/netip"
"sync"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"github.com/netbirdio/netbird/client/firewall"
fwmgr "github.com/netbirdio/netbird/client/firewall/manager"
"github.com/netbirdio/netbird/client/iface"
"github.com/netbirdio/netbird/client/iface/wgaddr"
"github.com/netbirdio/netbird/client/internal/acl/mocks"
mgmProto "github.com/netbirdio/netbird/shared/management/proto"
)
// sourcesRecordingFirewall wraps a real firewall.Manager and records
// the source prefixes of every AddFilterRule call.
type sourcesRecordingFirewall struct {
fwmgr.Manager
mu sync.Mutex
sources [][]netip.Prefix
}
func (f *sourcesRecordingFirewall) AddFilterRule(id []byte, sources []netip.Prefix, destination fwmgr.Network, proto fwmgr.Protocol, sPort, dPort *fwmgr.Port, action fwmgr.Action) (fwmgr.Rule, error) {
f.mu.Lock()
f.sources = append(f.sources, sources)
f.mu.Unlock()
return f.Manager.AddFilterRule(id, sources, destination, proto, sPort, dPort, action)
}
// TestLegacyManagementFallbackUsesMatchAnySources verifies the
// allow-all fallback for old management servers (empty FirewallRules
// without the FirewallRulesIsEmpty flag) reaches the firewall as /0
// match-any sources. The fallback rule carries PeerIP 0.0.0.0; if that
// were converted to a host prefix (0.0.0.0/32) it would match nothing
// and all peer traffic would be dropped.
func TestLegacyManagementFallbackUsesMatchAnySources(t *testing.T) {
t.Setenv("NB_WG_KERNEL_DISABLED", "true")
t.Setenv(firewall.EnvForceUserspaceFirewall, "true")
ctrl := gomock.NewController(t)
defer ctrl.Finish()
ifaceMock := mocks.NewMockIFaceMapper(ctrl)
ifaceMock.EXPECT().IsUserspaceBind().Return(true).AnyTimes()
ifaceMock.EXPECT().SetFilter(gomock.Any())
network := netip.MustParsePrefix("172.0.0.1/32")
ifaceMock.EXPECT().Name().Return("lo").AnyTimes()
ifaceMock.EXPECT().Address().Return(wgaddr.Address{IP: network.Addr(), Network: network}).AnyTimes()
ifaceMock.EXPECT().GetWGDevice().Return(nil).AnyTimes()
realFW, err := firewall.NewFirewall(ifaceMock, nil, flowLogger, false, iface.DefaultMTU)
require.NoError(t, err)
defer func() { require.NoError(t, realFW.Close(nil)) }()
fw := &sourcesRecordingFirewall{Manager: realFW}
acl := NewDefaultManager(fw)
// Old management: no rules and no FirewallRulesIsEmpty flag.
acl.ApplyFiltering(&mgmProto.NetworkMap{FirewallRules: nil, FirewallRulesIsEmpty: false}, false)
fw.mu.Lock()
defer fw.mu.Unlock()
require.NotEmpty(t, fw.sources, "legacy fallback must install at least one allow-all rule")
for _, sources := range fw.sources {
require.NotEmpty(t, sources)
for _, p := range sources {
assert.Equal(t, 0, p.Bits(), "legacy fallback source %s must be a /0 match-any prefix", p)
}
}
}
+360 -191
View File
@@ -1,8 +1,6 @@
package acl
import (
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"net/netip"
@@ -24,6 +22,10 @@ import (
var ErrSourceRangesEmpty = errors.New("sources range is empty")
// ErrNoRuleReturned is returned when the firewall backend reports success
// from AddFilterRule but yields no rule to track.
var ErrNoRuleReturned = errors.New("backend returned no rule")
// Manager is a ACL rules manager
type Manager interface {
ApplyFiltering(networkMap *mgmProto.NetworkMap, dnsRouteFeatureFlag bool)
@@ -32,19 +34,48 @@ type Manager interface {
// DefaultManager uses firewall manager to handle
type DefaultManager struct {
firewall firewall.Manager
ipsetCounter int
peerRulesPairs map[id.RuleID][]firewall.Rule
routeRules map[id.RuleID]struct{}
routeRules map[id.RuleID]firewall.Rule
previousConfigHash uint64
hasAppliedConfig bool
mutex sync.Mutex
}
// peerRuleGroup collapses a set of single-source FirewallRules sharing
// the same selector into one multi-source rule to push to the backend.
type peerRuleGroup struct {
direction mgmProto.RuleDirection
action mgmProto.RuleAction
protocol mgmProto.RuleProtocol
port *mgmProto.PortInfo
// legacyPort is used only when PortInfo is empty (old management).
legacyPort string
policyID []byte
sources []netip.Prefix
}
// peerRuleKey is the comparable selector that decides which single-source
// rules merge into one group. Rules with an equal key collapse into one
// multi-source backend rule. PortInfo is flattened into its scalar fields
// so the key compares by value; policyID keeps policies separate so two
// policies authorizing different peers don't merge under one attribution.
type peerRuleKey struct {
v6 bool
policyID string
direction mgmProto.RuleDirection
action mgmProto.RuleAction
protocol mgmProto.RuleProtocol
legacyPort string
port uint16
rangeStart uint16
rangeEnd uint16
}
func NewDefaultManager(fm firewall.Manager) *DefaultManager {
return &DefaultManager{
firewall: fm,
peerRulesPairs: make(map[id.RuleID][]firewall.Rule),
routeRules: make(map[id.RuleID]struct{}),
routeRules: make(map[id.RuleID]firewall.Rule),
}
}
@@ -88,11 +119,14 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap, dnsRout
time.Since(start), total)
}()
d.applyPeerACLs(networkMap)
peerErr := d.applyPeerACLs(networkMap)
if peerErr != nil {
log.Errorf("apply peer ACLs: %v", peerErr)
}
routeErr := d.applyRouteACLs(networkMap.RoutesFirewallRules, dnsRouteFeatureFlag)
if routeErr != nil {
log.Errorf("Failed to apply route ACLs: %v", routeErr)
log.Errorf("apply route ACLs: %v", routeErr)
}
flushErr := d.firewall.Flush()
@@ -104,7 +138,7 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap, dnsRout
// If applying or flushing failed, leave the previous hash untouched so the
// next (possibly identical) update is not skipped and gets a chance to
// reconcile the firewall state.
if err == nil && routeErr == nil && flushErr == nil {
if err == nil && peerErr == nil && routeErr == nil && flushErr == nil {
d.previousConfigHash = hash
d.hasAppliedConfig = true
} else {
@@ -135,7 +169,7 @@ func (d *DefaultManager) firewallConfigHash(networkMap *mgmProto.NetworkMap, dns
})
}
func (d *DefaultManager) applyPeerACLs(networkMap *mgmProto.NetworkMap) {
func (d *DefaultManager) applyPeerACLs(networkMap *mgmProto.NetworkMap) error {
rules := networkMap.FirewallRules
// if we got empty rules list but management not set networkMap.FirewallRulesIsEmpty flag
@@ -158,59 +192,167 @@ func (d *DefaultManager) applyPeerACLs(networkMap *mgmProto.NetworkMap) {
)
}
newRulePairs := make(map[id.RuleID][]firewall.Rule)
ipsetByRuleSelectors := make(map[string]string)
// Group incoming single-source rules from management by their
// (direction, action, proto, port) selector and merge sources.
// One call to the firewall backend per merged rule.
// A deny we cannot decode would leave its traffic unblocked, so skip
// the whole pass and keep existing rules until the next sync.
groups, denyErr, err := groupPeerRules(rules)
if denyErr != nil {
return fmt.Errorf("decode deny rule sources: %w", denyErr)
}
// TODO: deny rules should be fatal: if a deny rule fails to apply, we must
// roll back all allow rules to avoid a fail-open where allowed traffic bypasses
// the missing deny. Currently we accumulate errors and continue.
newRulePairs := make(map[id.RuleID][]firewall.Rule)
var merr *multierror.Error
for _, r := range rules {
// if this rule is member of rule selection with more than DefaultIPsCountForSet
// it's IP address can be used in the ipset for firewall manager which supports it
selector := d.getRuleGroupingSelector(r)
ipsetName, ok := ipsetByRuleSelectors[selector]
if !ok {
d.ipsetCounter++
ipsetName = fmt.Sprintf("nb%07d", d.ipsetCounter)
ipsetByRuleSelectors[selector] = ipsetName
}
pairID, rulePair, err := d.protoRuleToFirewallRule(r, ipsetName)
if err != nil {
merr = multierror.Append(merr, fmt.Errorf("apply firewall rule: %w", err))
if err != nil {
merr = multierror.Append(merr, err)
}
// Apply denies first. A deny that fails to install is a security
// failure (fail-open), so if any deny errors we roll back the
// denies we already installed in this pass and bail out without
// installing any accept. Pre-existing rules stay untouched until
// the next successful pass clears them.
denies, accepts := splitDenyAccept(groups)
if err := d.installPeerGroups(denies, newRulePairs, true); err != nil {
return fmt.Errorf("install deny rules: %w", err)
}
if err := d.installPeerGroups(accepts, newRulePairs, false); err != nil {
merr = multierror.Append(merr, err)
}
// Tear down rules that disappeared from the networkmap. Any rule
// the backend refuses to delete stays in our tracking so it gets
// retried on the next ApplyFiltering. Otherwise a transient
// delete failure would leak the rule in the firewall until the
// process exits.
for pairID, rules := range d.peerRulesPairs {
if _, ok := newRulePairs[pairID]; ok {
continue
}
if len(rulePair) > 0 {
d.peerRulesPairs[pairID] = rulePair
newRulePairs[pairID] = rulePair
}
}
if merr != nil {
log.Errorf("failed to apply %d peer ACL rule(s): %v", merr.Len(), nberrors.FormatErrorOrNil(merr))
}
for pairID, rules := range d.peerRulesPairs {
if _, ok := newRulePairs[pairID]; !ok {
for _, rule := range rules {
if err := d.firewall.DeletePeerRule(rule); err != nil {
log.Errorf("failed to delete peer firewall rule: %v", err)
continue
}
var remaining []firewall.Rule
for _, rule := range rules {
if err := d.firewall.DeleteFilterRule(rule); err != nil {
merr = multierror.Append(merr, fmt.Errorf("delete peer rule, will retry: %w", err))
remaining = append(remaining, rule)
}
delete(d.peerRulesPairs, pairID)
}
if len(remaining) > 0 {
newRulePairs[pairID] = remaining
}
}
d.peerRulesPairs = newRulePairs
return nberrors.FormatErrorOrNil(merr)
}
// installPeerGroups applies each group and records the resulting rule
// pairs in newRulePairs. With atomic set (deny rules), a single failure
// rolls back every rule installed in this call and returns, leaving the
// firewall exactly as before: denies are fail-closed and must be applied
// all-or-nothing. With atomic unset (accept rules), failures are
// accumulated and the remaining groups still install, so one malformed
// allow cannot drop every other legitimate allow in the pass.
func (d *DefaultManager) installPeerGroups(groups []*peerRuleGroup, newRulePairs map[id.RuleID][]firewall.Rule, atomic bool) error {
var freshlyInstalled []id.RuleID
var merr *multierror.Error
for _, g := range groups {
pairID, rulePair, err := d.applyPeerGroup(g)
if err != nil {
if atomic {
d.rollbackInstalled(freshlyInstalled)
return fmt.Errorf("apply firewall rule: %w", err)
}
merr = multierror.Append(merr, fmt.Errorf("apply firewall rule: %w", err))
continue
}
if len(rulePair) == 0 {
continue
}
if _, existed := d.peerRulesPairs[pairID]; !existed {
freshlyInstalled = append(freshlyInstalled, pairID)
}
d.peerRulesPairs[pairID] = rulePair
newRulePairs[pairID] = rulePair
}
return nberrors.FormatErrorOrNil(merr)
}
func (d *DefaultManager) rollbackInstalled(pairIDs []id.RuleID) {
var merr *multierror.Error
for _, pairID := range pairIDs {
// Keep any rule the backend refuses to delete tracked so it is
// retried on the next ApplyFiltering instead of leaking in the
// firewall with no tracking left to remove it.
var remaining []firewall.Rule
for _, rule := range d.peerRulesPairs[pairID] {
if err := d.firewall.DeleteFilterRule(rule); err != nil {
merr = multierror.Append(merr, fmt.Errorf("rule %s: %w", pairID, err))
remaining = append(remaining, rule)
}
}
if len(remaining) > 0 {
d.peerRulesPairs[pairID] = remaining
} else {
delete(d.peerRulesPairs, pairID)
}
}
if err := nberrors.FormatErrorOrNil(merr); err != nil {
log.Errorf("rollback peer rules: %v", err)
}
}
func (d *DefaultManager) applyPeerGroup(g *peerRuleGroup) (id.RuleID, []firewall.Rule, error) {
protocol, err := ConvertToFirewallProtocol(g.protocol)
if err != nil {
return "", nil, fmt.Errorf("skipping firewall rule: %w", err)
}
action, err := convertFirewallAction(g.action)
if err != nil {
return "", nil, fmt.Errorf("skipping firewall rule: %w", err)
}
port, err := resolveGroupPort(g)
if err != nil {
return "", nil, err
}
var fwRule firewall.Rule
switch g.direction {
case mgmProto.RuleDirection_IN:
fwRule, err = d.firewall.AddFilterRule(g.policyID, g.sources, firewall.Network{}, protocol, nil, port, action)
case mgmProto.RuleDirection_OUT:
if d.firewall.IsStateful() {
return "", nil, nil
}
if shouldSkipInvertedRule(protocol, port) {
return "", nil, nil
}
fwRule, err = d.firewall.AddFilterRule(g.policyID, g.sources, firewall.Network{}, protocol, port, nil, action)
default:
return "", nil, errors.New("invalid direction")
}
if err != nil {
return "", nil, fmt.Errorf("add firewall rule: %w", err)
}
if fwRule == nil {
return "", nil, fmt.Errorf("add firewall rule: %w", ErrNoRuleReturned)
}
// Derive the pair id from the backend rule, like the route path:
// the backend dedups identical content, so two policies authorizing
// the same flow resolve to the same id and a single backing rule.
return fwRule.ID(), []firewall.Rule{fwRule}, nil
}
func (d *DefaultManager) applyRouteACLs(rules []*mgmProto.RouteFirewallRule, dynamicResolver bool) error {
newRouteRules := make(map[id.RuleID]struct{}, len(rules))
newRouteRules := make(map[id.RuleID]firewall.Rule, len(rules))
var merr *multierror.Error
// Apply new rules - firewall manager will return existing rule ID if already present
// Apply new rules - firewall manager will return the existing rule if already present
for _, rule := range rules {
id, err := d.applyRouteACL(rule, dynamicResolver)
addedRule, err := d.applyRouteACL(rule, dynamicResolver)
if err != nil {
if errors.Is(err, ErrSourceRangesEmpty) {
log.Debugf("skipping empty sources rule with destination %s: %v", rule.Destination, err)
@@ -219,16 +361,18 @@ func (d *DefaultManager) applyRouteACLs(rules []*mgmProto.RouteFirewallRule, dyn
}
continue
}
newRouteRules[id] = struct{}{}
newRouteRules[addedRule.ID()] = addedRule
}
// Clean up old firewall rules
for id := range d.routeRules {
if _, exists := newRouteRules[id]; !exists {
if err := d.firewall.DeleteRouteRule(id); err != nil {
merr = multierror.Append(merr, fmt.Errorf("delete route rule: %w", err))
}
// implicitly deleted from the map
// Tear down old route rules; retain ones the backend refused so a
// transient failure doesn't leave orphaned rules in the firewall.
for ruleID, rule := range d.routeRules {
if _, exists := newRouteRules[ruleID]; exists {
continue
}
if err := d.firewall.DeleteFilterRule(rule); err != nil {
merr = multierror.Append(merr, fmt.Errorf("delete route rule, will retry: %w", err))
newRouteRules[ruleID] = rule
}
}
@@ -236,102 +380,202 @@ func (d *DefaultManager) applyRouteACLs(rules []*mgmProto.RouteFirewallRule, dyn
return nberrors.FormatErrorOrNil(merr)
}
func (d *DefaultManager) applyRouteACL(rule *mgmProto.RouteFirewallRule, dynamicResolver bool) (id.RuleID, error) {
func (d *DefaultManager) applyRouteACL(rule *mgmProto.RouteFirewallRule, dynamicResolver bool) (firewall.Rule, error) {
if len(rule.SourceRanges) == 0 {
return "", ErrSourceRangesEmpty
return nil, ErrSourceRangesEmpty
}
var sources []netip.Prefix
for _, sourceRange := range rule.SourceRanges {
source, err := netip.ParsePrefix(sourceRange)
if err != nil {
return "", fmt.Errorf("parse source range: %w", err)
return nil, fmt.Errorf("parse source range: %w", err)
}
sources = append(sources, source)
sources = append(sources, firewall.UnmapPrefix(source))
}
destination, err := determineDestination(rule, dynamicResolver, sources)
if err != nil {
return "", fmt.Errorf("determine destination: %w", err)
return nil, fmt.Errorf("determine destination: %w", err)
}
protocol, err := convertToFirewallProtocol(rule.Protocol)
protocol, err := ConvertToFirewallProtocol(rule.Protocol)
if err != nil {
return "", fmt.Errorf("invalid protocol: %w", err)
return nil, fmt.Errorf("invalid protocol: %w", err)
}
action, err := convertFirewallAction(rule.Action)
if err != nil {
return "", fmt.Errorf("invalid action: %w", err)
return nil, fmt.Errorf("invalid action: %w", err)
}
dPorts := convertPortInfo(rule.PortInfo)
addedRule, err := d.firewall.AddRouteFiltering(rule.PolicyID, sources, destination, protocol, nil, dPorts, action)
addedRule, err := d.firewall.AddFilterRule(rule.PolicyID, sources, destination, protocol, nil, dPorts, action)
if err != nil {
return "", fmt.Errorf("add route rule: %w", err)
return nil, fmt.Errorf("add route rule: %w", err)
}
if addedRule == nil {
return nil, fmt.Errorf("add route rule: %w", ErrNoRuleReturned)
}
return id.RuleID(addedRule.ID()), nil
return addedRule, nil
}
func (d *DefaultManager) protoRuleToFirewallRule(
r *mgmProto.FirewallRule,
ipsetName string,
) (id.RuleID, []firewall.Rule, error) {
ip, err := extractRuleIP(r)
if err != nil {
return "", nil, err
// splitDenyAccept partitions groups by action so denies can be
// applied before accepts. Order within each bucket is preserved.
func splitDenyAccept(groups []*peerRuleGroup) (denies, accepts []*peerRuleGroup) {
for _, g := range groups {
if g.action == mgmProto.RuleAction_DROP {
denies = append(denies, g)
} else {
accepts = append(accepts, g)
}
}
return denies, accepts
}
// groupPeerRules merges single-source rules sharing a selector into
// multi-source groups. It splits source-decode failures by action:
// denyErr is non-nil when a deny rule could not be decoded, which is a
// fail-open risk the caller must treat as fatal for the pass; err
// carries the tolerable accept-rule failures the caller can log and
// continue past.
func groupPeerRules(rules []*mgmProto.FirewallRule) (groups []*peerRuleGroup, denyErr error, err error) {
var denyMerr, acceptMerr *multierror.Error
byKey := make(map[peerRuleKey]*peerRuleGroup)
order := make([]peerRuleKey, 0)
for _, r := range rules {
srcs, decErr := extractRuleSources(r)
if decErr != nil {
if r.Action == mgmProto.RuleAction_DROP {
denyMerr = multierror.Append(denyMerr, decErr)
} else {
acceptMerr = multierror.Append(acceptMerr, decErr)
}
continue
}
// A single FirewallRule normally carries one address family, but
// split by family defensively: each backend keys a rule to one
// family and would mismatch sources of the other, so a group's
// sources must never span families.
v4, v6 := splitPrefixesByFamily(srcs)
for _, sub := range []struct {
isV6 bool
sources []netip.Prefix
}{{false, v4}, {true, v6}} {
if len(sub.sources) == 0 {
continue
}
key := ruleGroupKey(r, sub.isV6)
g, ok := byKey[key]
if !ok {
g = &peerRuleGroup{
direction: r.Direction,
action: r.Action,
protocol: r.Protocol,
port: r.PortInfo,
legacyPort: r.Port,
policyID: r.PolicyID,
}
byKey[key] = g
order = append(order, key)
}
g.sources = append(g.sources, sub.sources...)
}
}
protocol, err := convertToFirewallProtocol(r.Protocol)
if err != nil {
return "", nil, fmt.Errorf("skipping firewall rule: %s", err)
out := make([]*peerRuleGroup, 0, len(order))
for _, k := range order {
out = append(out, byKey[k])
}
return out, nberrors.FormatErrorOrNil(denyMerr), nberrors.FormatErrorOrNil(acceptMerr)
}
func prefixIsV6(p netip.Prefix) bool {
return p.Addr().Is6() && !p.Addr().Is4In6()
}
// splitPrefixesByFamily partitions prefixes into IPv4 and IPv6 groups.
func splitPrefixesByFamily(prefixes []netip.Prefix) (v4, v6 []netip.Prefix) {
for _, p := range prefixes {
if prefixIsV6(p) {
v6 = append(v6, p)
} else {
v4 = append(v4, p)
}
}
return v4, v6
}
// ruleGroupKey builds the selector key for a rule. v6 must reflect the
// rule's source family: mgmt emits one rule per family and mixing them
// would break ICMP-variant selection in uspfilter.
func ruleGroupKey(r *mgmProto.FirewallRule, v6 bool) peerRuleKey {
k := peerRuleKey{
v6: v6,
policyID: string(r.PolicyID),
direction: r.Direction,
action: r.Action,
protocol: r.Protocol,
legacyPort: r.Port,
}
if pi := r.PortInfo; pi != nil {
k.port = uint16(pi.GetPort())
if rng := pi.GetRange(); rng != nil {
k.rangeStart = uint16(rng.GetStart())
k.rangeEnd = uint16(rng.GetEnd())
}
}
return k
}
// extractRuleSources returns all source prefixes the rule applies to.
// New management populates sourcePrefixes; older management sets PeerIP.
func extractRuleSources(r *mgmProto.FirewallRule) ([]netip.Prefix, error) {
if len(r.SourcePrefixes) > 0 {
out := make([]netip.Prefix, 0, len(r.SourcePrefixes))
for _, raw := range r.SourcePrefixes {
addr, err := netiputil.DecodeAddr(raw)
if err != nil {
return nil, fmt.Errorf("decode source prefix: %w", err)
}
out = append(out, netip.PrefixFrom(addr.Unmap(), addr.Unmap().BitLen()))
}
return out, nil
}
action, err := convertFirewallAction(r.Action)
peerIP := r.PeerIP //nolint:staticcheck // PeerIP is the legacy source field for old management servers
addr, err := netip.ParseAddr(peerIP)
if err != nil {
return "", nil, fmt.Errorf("skipping firewall rule: %s", err)
return nil, fmt.Errorf("parse peer IP %q: %w", peerIP, err)
}
addr = addr.Unmap()
// An unspecified PeerIP means "any peer" (legacy management
// allow-all fallback); only a /0 prefix matches any source in the
// backends, a full-length prefix would match nothing.
if addr.IsUnspecified() {
return []netip.Prefix{netip.PrefixFrom(addr, 0)}, nil
}
return []netip.Prefix{netip.PrefixFrom(addr, addr.BitLen())}, nil
}
var port *firewall.Port
if !portInfoEmpty(r.PortInfo) {
port = convertPortInfo(r.PortInfo)
} else if r.Port != "" {
// old version of management, single port
value, err := strconv.Atoi(r.Port)
func resolveGroupPort(g *peerRuleGroup) (*firewall.Port, error) {
if !portInfoEmpty(g.port) {
return convertPortInfo(g.port), nil
}
if g.legacyPort != "" {
value, err := strconv.ParseUint(g.legacyPort, 10, 16)
if err != nil {
return "", nil, fmt.Errorf("invalid port: %w", err)
return nil, fmt.Errorf("invalid port: %w", err)
}
port = &firewall.Port{
return &firewall.Port{
Values: []uint16{uint16(value)},
}
}, nil
}
ruleID := d.getPeerRuleID(ip, protocol, int(r.Direction), port, action)
if rulesPair, ok := d.peerRulesPairs[ruleID]; ok {
return ruleID, rulesPair, nil
}
var rules []firewall.Rule
switch r.Direction {
case mgmProto.RuleDirection_IN:
rules, err = d.addInRules(r.PolicyID, ip, protocol, port, action, ipsetName)
case mgmProto.RuleDirection_OUT:
if d.firewall.IsStateful() {
return "", nil, nil
}
// return traffic for outbound connections if firewall is stateless
rules, err = d.addOutRules(r.PolicyID, ip, protocol, port, action, ipsetName)
default:
return "", nil, fmt.Errorf("invalid direction, skipping firewall rule")
}
if err != nil {
return "", nil, err
}
return ruleID, rules, nil
// nolint:nilnil // a nil port legitimately means "no port restriction"
return nil, nil
}
func portInfoEmpty(portInfo *mgmProto.PortInfo) bool {
@@ -350,84 +594,9 @@ func portInfoEmpty(portInfo *mgmProto.PortInfo) bool {
}
}
func (d *DefaultManager) addInRules(
id []byte,
ip netip.Addr,
protocol firewall.Protocol,
port *firewall.Port,
action firewall.Action,
ipsetName string,
) ([]firewall.Rule, error) {
rule, err := d.firewall.AddPeerFiltering(id, ip.AsSlice(), protocol, nil, port, action, ipsetName)
if err != nil {
return nil, fmt.Errorf("add firewall rule: %w", err)
}
return rule, nil
}
func (d *DefaultManager) addOutRules(
id []byte,
ip netip.Addr,
protocol firewall.Protocol,
port *firewall.Port,
action firewall.Action,
ipsetName string,
) ([]firewall.Rule, error) {
if shouldSkipInvertedRule(protocol, port) {
return nil, nil
}
rule, err := d.firewall.AddPeerFiltering(id, ip.AsSlice(), protocol, port, nil, action, ipsetName)
if err != nil {
return nil, fmt.Errorf("add firewall rule: %w", err)
}
return rule, nil
}
// getPeerRuleID returns unique ID for the rule based on its parameters.
func (d *DefaultManager) getPeerRuleID(
ip netip.Addr,
proto firewall.Protocol,
direction int,
port *firewall.Port,
action firewall.Action,
) id.RuleID {
idStr := ip.String() + string(proto) + strconv.Itoa(direction) + strconv.Itoa(int(action))
if port != nil {
idStr += port.String()
}
return id.RuleID(hex.EncodeToString(md5.New().Sum([]byte(idStr))))
}
// getRuleGroupingSelector takes all rule properties except IP address to build selector
func (d *DefaultManager) getRuleGroupingSelector(rule *mgmProto.FirewallRule) string {
return fmt.Sprintf("%v:%v:%v:%s:%v", strconv.Itoa(int(rule.Direction)), rule.Action, rule.Protocol, rule.Port, rule.PortInfo)
}
// extractRuleIP extracts the peer IP from a firewall rule.
// If sourcePrefixes is populated (new management), decode the first entry and use its address.
// Otherwise fall back to the deprecated PeerIP string field (old management).
func extractRuleIP(r *mgmProto.FirewallRule) (netip.Addr, error) {
if len(r.SourcePrefixes) > 0 {
addr, err := netiputil.DecodeAddr(r.SourcePrefixes[0])
if err != nil {
return netip.Addr{}, fmt.Errorf("decode source prefix: %w", err)
}
return addr.Unmap(), nil
}
//nolint:staticcheck // PeerIP used for backward compatibility with old management
addr, err := netip.ParseAddr(r.PeerIP)
if err != nil {
return netip.Addr{}, fmt.Errorf("invalid IP address, skipping firewall rule")
}
return addr.Unmap(), nil
}
func convertToFirewallProtocol(protocol mgmProto.RuleProtocol) (firewall.Protocol, error) {
// ConvertToFirewallProtocol maps a management rule protocol to the
// firewall protocol type.
func ConvertToFirewallProtocol(protocol mgmProto.RuleProtocol) (firewall.Protocol, error) {
switch protocol {
case mgmProto.RuleProtocol_TCP:
return firewall.ProtocolTCP, nil
+4 -3
View File
@@ -10,6 +10,7 @@ import (
"go.uber.org/mock/gomock"
"github.com/netbirdio/netbird/client/firewall"
fwmanager "github.com/netbirdio/netbird/client/firewall/manager"
"github.com/netbirdio/netbird/client/iface"
"github.com/netbirdio/netbird/client/iface/wgaddr"
"github.com/netbirdio/netbird/client/internal/acl/mocks"
@@ -77,9 +78,9 @@ func TestDefaultManager(t *testing.T) {
})
t.Run("add extra rules", func(t *testing.T) {
existedPairs := map[string]struct{}{}
existedPairs := map[fwmanager.RuleID]struct{}{}
for id := range acl.peerRulesPairs {
existedPairs[id.ID()] = struct{}{}
existedPairs[id] = struct{}{}
}
// remove first rule
@@ -106,7 +107,7 @@ func TestDefaultManager(t *testing.T) {
// check that old rule was removed
previousCount := 0
for id := range acl.peerRulesPairs {
if _, ok := existedPairs[id.ID()]; ok {
if _, ok := existedPairs[id]; ok {
previousCount++
}
}
+1
View File
@@ -372,6 +372,7 @@ func (a *Auth) setSystemInfoFlags(info *system.Info) {
a.config.EnableSSHLocalPortForwarding,
a.config.EnableSSHRemotePortForwarding,
a.config.DisableSSHAuth,
a.config.RemoteJobsAllowed,
)
}
+10
View File
@@ -304,6 +304,16 @@ func (d *DeviceAuthorizationFlow) WaitToken(ctx context.Context, info AuthFlowIn
return TokenInfo{}, fmt.Errorf("validate access token failed with error: %v", err)
}
// Same as the PKCE flow: the account the token belongs to is what
// callers store to send back as the login_hint. Without it a client
// driven through the device flow — Android TV and tvOS — never binds
// an account to its profile and every later login goes out blind.
if email, err := parseEmailFromIDToken(tokenInfo.IDToken); err != nil {
log.Warnf("failed to parse email from ID token: %v", err)
} else {
tokenInfo.Email = email
}
log.Infof("device flow: user authorization confirmed after %d polls in %s", polls, time.Since(start).Round(time.Second))
return tokenInfo, err
}
+3 -1
View File
@@ -242,7 +242,7 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan
wrapErr := state.Wrap
myPrivateKey, err := wgtypes.ParseKey(c.config.PrivateKey)
if err != nil {
log.Errorf("failed parsing Wireguard key %s: [%s]", c.config.PrivateKey, err.Error())
log.Errorf("failed parsing Wireguard key: %s", err)
return wrapErr(err)
}
@@ -652,6 +652,7 @@ func createEngineConfig(key wgtypes.Key, config *profilemanager.Config, peerConf
RosenpassEnabled: config.RosenpassEnabled,
RosenpassPermissive: config.RosenpassPermissive,
ServerSSHAllowed: util.ReturnBoolWithDefaultTrue(config.ServerSSHAllowed),
RemoteJobsAllowed: util.ReturnBoolWithDefaultFalse(config.RemoteJobsAllowed),
EnableSSHRoot: config.EnableSSHRoot,
EnableSSHSFTP: config.EnableSSHSFTP,
EnableSSHLocalPortForwarding: config.EnableSSHLocalPortForwarding,
@@ -749,6 +750,7 @@ func loginToManagement(ctx context.Context, client mgm.Client, pubSSHKey []byte,
config.EnableSSHLocalPortForwarding,
config.EnableSSHRemotePortForwarding,
config.DisableSSHAuth,
config.RemoteJobsAllowed,
)
return client.Login(sysInfo, pubSSHKey, config.DNSLabels)
}
+71 -58
View File
@@ -5,65 +5,78 @@ import (
"testing"
)
func Test_freePort(t *testing.T) {
tests := []struct {
name string
port int
want int
shouldMatch bool
}{
{
name: "when port is 0 use random port",
port: 0,
want: 0,
shouldMatch: false,
},
{
name: "provided and available",
port: 51821,
want: 51821,
shouldMatch: true,
},
{
name: "provided and not available",
port: 51830,
want: 51830,
shouldMatch: false,
},
}
c1, err := net.ListenUDP("udp", &net.UDPAddr{Port: 0})
// probeFreePort asks the OS for a free UDP port and immediately releases it.
// The returned number is only a hint: nothing stops another process from
// grabbing the same port before the caller gets a chance to bind it.
//
// A hardcoded port number is not an option here: any fixed number can fall
// inside the ephemeral range and be held by an unrelated process on the test
// runner.
func probeFreePort(t *testing.T) int {
t.Helper()
conn, err := net.ListenUDP("udp", &net.UDPAddr{Port: 0})
if err != nil {
t.Errorf("freePort error = %v", err)
t.Fatalf("failed to bind probe port: %v", err)
}
defer func(c1 *net.UDPConn) {
_ = c1.Close()
}(c1)
if tests[1].port == c1.LocalAddr().(*net.UDPAddr).Port {
tests[1].port++
tests[1].want++
}
tests[2].port = c1.LocalAddr().(*net.UDPAddr).Port
tests[2].want = c1.LocalAddr().(*net.UDPAddr).Port
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := freePort(tt.port)
if err != nil {
t.Errorf("got an error while getting free port: %v", err)
}
if tt.shouldMatch && got != tt.want {
t.Errorf("got a different port %v, want %v", got, tt.want)
}
if !tt.shouldMatch && got == tt.want {
t.Errorf("got the same port %v, want a different port", tt.want)
}
})
port := conn.LocalAddr().(*net.UDPAddr).Port
if err := conn.Close(); err != nil {
t.Fatalf("failed to close probe port: %v", err)
}
return port
}
func Test_freePort(t *testing.T) {
t.Run("when port is 0 use random port", func(t *testing.T) {
got, err := freePort(0)
if err != nil {
t.Fatalf("got an error while getting free port: %v", err)
}
if got == 0 {
t.Errorf("got port 0, want a non-zero random port")
}
})
t.Run("provided and available", func(t *testing.T) {
const maxAttempts = 5
// The probed port is released before freePort binds it, so an
// unrelated process on the test runner can grab it in between,
// making freePort fall back to a different port. Retry with a
// freshly probed port instead of failing on a lost race.
for attempt := 1; attempt <= maxAttempts; attempt++ {
candidate := probeFreePort(t)
got, err := freePort(candidate)
if err != nil {
t.Fatalf("got an error while getting free port: %v", err)
}
if got == candidate {
return
}
t.Logf("attempt %d: freePort returned %d instead of the requested %d, retrying", attempt, got, candidate)
}
t.Fatalf("freePort did not return the requested free port after %d attempts", maxAttempts)
})
t.Run("provided and not available", func(t *testing.T) {
busy, err := net.ListenUDP("udp", &net.UDPAddr{Port: 0})
if err != nil {
t.Fatalf("failed to bind busy port: %v", err)
}
t.Cleanup(func() {
_ = busy.Close()
})
busyPort := busy.LocalAddr().(*net.UDPAddr).Port
got, err := freePort(busyPort)
if err != nil {
t.Fatalf("got an error while getting free port: %v", err)
}
if got == busyPort {
t.Errorf("got the same port %v, want a different port", busyPort)
}
})
}
+42
View File
@@ -0,0 +1,42 @@
package daemonaddr
import (
"os"
"strconv"
log "github.com/sirupsen/logrus"
)
const (
// EnvMaxRecvMsgSize overrides the default gRPC max receive message size for
// connections to the daemon. Value is in bytes.
EnvMaxRecvMsgSize = "NB_DAEMON_GRPC_MAX_MSG_SIZE"
// defaultMaxRecvMsgSize is the max gRPC receive message size used for daemon
// connections when EnvMaxRecvMsgSize is unset or invalid. It overrides the
// gRPC library default of 4 MB, which a detailed status already exceeds on a
// network of a few thousand peers.
defaultMaxRecvMsgSize = 1024 * 1024 * 16
)
// MaxRecvMsgSize returns the max gRPC receive message size for daemon connections
// from the environment, or defaultMaxRecvMsgSize (16 MB) if unset or invalid.
func MaxRecvMsgSize() int {
val := os.Getenv(EnvMaxRecvMsgSize)
if val == "" {
return defaultMaxRecvMsgSize
}
size, err := strconv.Atoi(val)
if err != nil {
log.Warnf("invalid %s value %q, using default: %v", EnvMaxRecvMsgSize, val, err)
return defaultMaxRecvMsgSize
}
if size <= 0 {
log.Warnf("invalid %s value %d, must be positive, using default", EnvMaxRecvMsgSize, size)
return defaultMaxRecvMsgSize
}
return size
}
+112
View File
@@ -0,0 +1,112 @@
package daemonaddr
import (
"context"
"net"
"os"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
"github.com/netbirdio/netbird/client/proto"
)
func TestMaxRecvMsgSize(t *testing.T) {
tests := []struct {
name string
envValue string
expected int
}{
{name: "unset returns default", envValue: "", expected: defaultMaxRecvMsgSize},
{name: "non-numeric returns default", envValue: "abc", expected: defaultMaxRecvMsgSize},
{name: "negative returns default", envValue: "-1", expected: defaultMaxRecvMsgSize},
{name: "zero returns default", envValue: "0", expected: defaultMaxRecvMsgSize},
{name: "valid value is used", envValue: "33554432", expected: 33554432},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
// Set first so the previous value is restored on cleanup, then unset to
// exercise the absent case.
t.Setenv(EnvMaxRecvMsgSize, tc.envValue)
if tc.envValue == "" {
require.NoError(t, os.Unsetenv(EnvMaxRecvMsgSize), "unset the override")
}
assert.Equal(t, tc.expected, MaxRecvMsgSize(), "max receive message size")
})
}
}
// bigStatusServer answers Status with a response larger than gRPC's 4 MB default
// receive limit, which is what a detailed status on a large network looks like.
type bigStatusServer struct {
proto.UnimplementedDaemonServiceServer
payload string
}
func (s *bigStatusServer) Status(context.Context, *proto.StatusRequest) (*proto.StatusResponse, error) {
return &proto.StatusResponse{Status: s.payload}, nil
}
func startBigStatusServer(t *testing.T, payload string) string {
t.Helper()
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err, "listen on loopback")
srv := grpc.NewServer()
proto.RegisterDaemonServiceServer(srv, &bigStatusServer{payload: payload})
go func() {
_ = srv.Serve(listener)
}()
t.Cleanup(srv.Stop)
return "tcp://" + listener.Addr().String()
}
func TestDialTargetAcceptsAStatusOverTheGrpcDefault(t *testing.T) {
payload := strings.Repeat("x", 5*1024*1024)
addr := startBigStatusServer(t, payload)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
target, opts := DialTarget(addr)
conn, err := grpc.NewClient(target, opts...)
require.NoError(t, err, "dial the daemon")
t.Cleanup(func() { _ = conn.Close() })
resp, err := proto.NewDaemonServiceClient(conn).Status(ctx, &proto.StatusRequest{})
require.NoError(t, err, "a detailed status must not be rejected for its size")
assert.Len(t, resp.GetStatus(), len(payload), "the whole response must arrive")
}
// TestDialTargetRaisesTheDefaultLimit is the negative control: the same response
// over a connection carrying gRPC's own defaults is refused, which is the failure
// reported by `netbird status -d` on a large deployment.
func TestDialTargetRaisesTheDefaultLimit(t *testing.T) {
payload := strings.Repeat("x", 5*1024*1024)
addr := startBigStatusServer(t, payload)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
conn, err := grpc.NewClient(
strings.TrimPrefix(addr, "tcp://"),
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
require.NoError(t, err, "dial with the library defaults")
t.Cleanup(func() { _ = conn.Close() })
_, err = proto.NewDaemonServiceClient(conn).Status(ctx, &proto.StatusRequest{})
require.Error(t, err, "the library default must reject this response")
assert.Equal(t, codes.ResourceExhausted, status.Code(err), "gRPC rejects an oversized message")
}
+4 -1
View File
@@ -36,7 +36,10 @@ const (
// address. The npipe scheme needs a context dialer because gRPC has no
// named-pipe resolver; unix and tcp are handled by gRPC itself.
func DialTarget(addr string) (string, []grpc.DialOption) {
opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(MaxRecvMsgSize())),
}
if name, ok := strings.CutPrefix(addr, pipeScheme); ok {
paths := PipePaths(name)
+58 -1
View File
@@ -379,9 +379,38 @@ func NewBundleGenerator(deps GeneratorDependencies, cfg BundleConfig) *BundleGen
}
}
// bundleFilePattern names the bundle zips Generate creates in tempDir; the
// asterisk is filled in by os.CreateTemp.
const bundleFilePattern = "netbird.debug.*.zip"
const exportedBundlePrefix = "netbird.debug-file."
const exportedBundleMaxAge = 24 * time.Hour
// RemoveStaleBundles deletes bundle zips that an interrupted generation or
// upload left behind in dir. Only files older than maxAge go, so a bundle that
// another caller is still writing or uploading in the same directory survives.
// Exported bundles are kept for exportedBundleMaxAge instead.
func RemoveStaleBundles(dir string, maxAge time.Duration) {
removeStaleFiles(dir, bundleFilePattern, maxAge)
removeStaleFiles(dir, exportedBundlePrefix+"*.zip", exportedBundleMaxAge)
}
// ExportBundle renames a generated bundle out of the RemoveStaleBundles pattern
// and returns the new path. The caller owns the file from then on; an export
// abandoned for longer than exportedBundleMaxAge is removed by RemoveStaleBundles.
func ExportBundle(path string) (string, error) {
base := strings.TrimPrefix(filepath.Base(path), strings.SplitN(bundleFilePattern, "*", 2)[0])
exported := filepath.Join(filepath.Dir(path), exportedBundlePrefix+base)
if err := os.Rename(path, exported); err != nil {
return "", fmt.Errorf("export debug bundle: %w", err)
}
return exported, nil
}
// Generate creates a debug bundle and returns the location.
func (g *BundleGenerator) Generate() (resp string, err error) {
bundlePath, err := os.CreateTemp(g.tempDir, "netbird.debug.*.zip")
bundlePath, err := os.CreateTemp(g.tempDir, bundleFilePattern)
if err != nil {
return "", fmt.Errorf("create zip file: %w", err)
}
@@ -711,6 +740,9 @@ func (g *BundleGenerator) addCommonConfigFields(configContent *strings.Builder)
if g.internalConfig.ServerSSHAllowed != nil {
configContent.WriteString(fmt.Sprintf("ServerSSHAllowed: %v\n", *g.internalConfig.ServerSSHAllowed))
}
if g.internalConfig.RemoteJobsAllowed != nil {
configContent.WriteString(fmt.Sprintf("RemoteJobsAllowed: %v\n", *g.internalConfig.RemoteJobsAllowed))
}
if g.internalConfig.EnableSSHRoot != nil {
configContent.WriteString(fmt.Sprintf("EnableSSHRoot: %v\n", *g.internalConfig.EnableSSHRoot))
}
@@ -737,6 +769,8 @@ func (g *BundleGenerator) addCommonConfigFields(configContent *strings.Builder)
configContent.WriteString(fmt.Sprintf("BlockLANAccess: %v\n", g.internalConfig.BlockLANAccess))
configContent.WriteString(fmt.Sprintf("BlockInbound: %v\n", g.internalConfig.BlockInbound))
configContent.WriteString(fmt.Sprintf("DisableIPv6: %v\n", g.internalConfig.DisableIPv6))
configContent.WriteString(fmt.Sprintf("LocalMetricsEnabled: %v\n", g.internalConfig.LocalMetricsEnabled))
configContent.WriteString(fmt.Sprintf("LocalMetricsAddress: %s\n", g.internalConfig.LocalMetricsAddress))
configContent.WriteString(fmt.Sprintf("SyncMessageVersion: %v\n", g.internalConfig.SyncMessageVersion))
if g.internalConfig.DisableNotifications != nil {
@@ -1720,3 +1754,26 @@ func anonymizeSlice(v []any, anonymizer *anonymize.Anonymizer) []any {
}
return v
}
func removeStaleFiles(dir, pattern string, maxAge time.Duration) {
matches, err := filepath.Glob(filepath.Join(dir, pattern))
if err != nil {
log.Debugf("glob stale debug bundles in %s: %v", dir, err)
return
}
cutoff := time.Now().Add(-maxAge)
for _, path := range matches {
info, err := os.Stat(path)
if err != nil || info.ModTime().After(cutoff) {
continue
}
if err := os.Remove(path); err != nil {
if !errors.Is(err, fs.ErrNotExist) {
log.Warnf("remove stale debug bundle %s: %v", path, err)
}
continue
}
log.Infof("removed stale debug bundle %s", path)
}
}
+66 -6
View File
@@ -4,6 +4,7 @@ import (
"archive/zip"
"bytes"
"encoding/json"
"fmt"
"net"
"net/netip"
"net/url"
@@ -839,12 +840,13 @@ COMMIT`
// the excluded set with a justification.
func TestAddConfig_AllFieldsCovered(t *testing.T) {
excluded := map[string]string{
"PrivateKey": "sensitive: WireGuard private key",
"PreSharedKey": "sensitive: WireGuard pre-shared key",
"SSHKey": "sensitive: SSH private key",
"ClientCertKeyPair": "non-config: parsed cert pair, not serialized",
"Name": "non-config: profile name is not needed for debug purposes",
"policy": "non-config: in-memory MDM policy snapshot, surfaced via Config.Policy() / GetConfigResponse.MDMManagedFields",
"PrivateKey": "sensitive: WireGuard private key",
"PreSharedKey": "sensitive: WireGuard pre-shared key",
"SSHKey": "sensitive: SSH private key",
"ClientCertKeyPair": "non-config: parsed cert pair, not serialized",
"Name": "non-config: profile name is not needed for debug purposes",
"policy": "non-config: in-memory MDM policy snapshot, surfaced via Config.Policy() / GetConfigResponse.MDMManagedFields",
"DebugBundleUploadURL": "sensitive: MDM-provided upload URL may carry credentials or query tokens; kept out of the shared bundle",
}
mURL, _ := url.Parse("https://api.example.com:443")
@@ -864,6 +866,7 @@ func TestAddConfig_AllFieldsCovered(t *testing.T) {
RosenpassEnabled: true,
RosenpassPermissive: true,
ServerSSHAllowed: &bTrue,
RemoteJobsAllowed: &bTrue,
EnableSSHRoot: &bTrue,
EnableSSHSFTP: &bTrue,
EnableSSHLocalPortForwarding: &bTrue,
@@ -886,6 +889,7 @@ func TestAddConfig_AllFieldsCovered(t *testing.T) {
ClientCertPath: "/tmp/cert",
ClientCertKeyPath: "/tmp/key",
LazyConnection: "on",
DebugBundleUploadURL: "https://upload.example.test/bundle?token=secret",
MTU: 1280,
DisableIPv6: true,
SyncMessageVersion: func(v int) *int { return &v }(1),
@@ -903,6 +907,13 @@ func TestAddConfig_AllFieldsCovered(t *testing.T) {
g.addCommonConfigFields(&sb)
rendered := sb.String() + renderAddConfigSpecific(g)
// DebugBundleUploadURL is an MDM-provided value that can carry
// credentials or signed query tokens. It is deliberately excluded
// above; assert it never reaches the rendered bundle — neither the
// field name nor the token — in either anonymize mode.
assert.NotContains(t, rendered, "DebugBundleUploadURL:", "MDM upload URL field must not be serialized into the debug bundle")
assert.NotContains(t, rendered, "token=secret", "MDM upload URL value must not leak into the debug bundle")
val := reflect.ValueOf(cfg).Elem()
typ := val.Type()
var missing []string
@@ -959,3 +970,52 @@ func renderAddConfigSpecific(g *BundleGenerator) string {
func newAnonymizerForTest() *anonymize.Anonymizer {
return anonymize.NewAnonymizer(anonymize.DefaultAddresses())
}
func TestRemoveStaleBundles(t *testing.T) {
dir := t.TempDir()
stale := filepath.Join(dir, "netbird.debug.111.zip")
fresh := filepath.Join(dir, "netbird.debug.222.zip")
other := filepath.Join(dir, "netbird.debug.333.txt")
owned := filepath.Join(dir, "netbird.debug.444.zip")
abandoned := filepath.Join(dir, "netbird.debug.555.zip")
for _, p := range []string{stale, fresh, other, owned, abandoned} {
require.NoError(t, os.WriteFile(p, []byte("x"), 0o600))
}
exported, err := ExportBundle(owned)
require.NoError(t, err)
exportedAbandoned, err := ExportBundle(abandoned)
require.NoError(t, err)
old := time.Now().Add(-2 * time.Hour)
for _, p := range []string{stale, other, exported} {
require.NoError(t, os.Chtimes(p, old, old))
}
ancient := time.Now().Add(-exportedBundleMaxAge - time.Hour)
require.NoError(t, os.Chtimes(exportedAbandoned, ancient, ancient))
RemoveStaleBundles(dir, time.Hour)
assert.NoFileExists(t, stale, "bundle older than maxAge should be removed")
assert.FileExists(t, fresh, "bundle younger than maxAge must survive, it may still be uploading")
assert.FileExists(t, other, "files outside the bundle pattern must not be touched")
assert.NoFileExists(t, owned)
assert.FileExists(t, exported, "exported bundle is caller-owned and must survive maxAge")
assert.NoFileExists(t, exportedAbandoned, "exported bundle older than exportedBundleMaxAge is abandoned")
}
func TestBundleIncludesNetworkMap(t *testing.T) {
for _, anonymize := range []bool{false, true} {
t.Run(fmt.Sprintf("anonymize=%t", anonymize), func(t *testing.T) {
g := NewBundleGenerator(GeneratorDependencies{
SyncResponse: &mgmProto.SyncResponse{NetworkMap: &mgmProto.NetworkMap{Serial: 1}},
}, BundleConfig{Anonymize: anonymize})
require.Contains(t, bundleEntries(t, g), "network_map.json")
})
}
}
func TestBundleOmitsNetworkMapWithoutSyncResponse(t *testing.T) {
g := NewBundleGenerator(GeneratorDependencies{}, BundleConfig{})
require.NotContains(t, bundleEntries(t, g), "network_map.json")
}
+206 -26
View File
@@ -6,8 +6,10 @@ import (
"fmt"
"io"
"net/netip"
"os"
"os/exec"
"slices"
"strconv"
"strings"
"syscall"
"time"
@@ -34,10 +36,16 @@ var (
// Registry locations of the host DNS configuration this package programs,
// exported so a diagnostic reader reports the same locations that are written.
const (
// NRPTKeyPrefix starts the name of every NRPT rule key this client creates.
// Older versions used different layouts under the same prefix: a single
// unsuffixed key, then one key per domain, now one key per batch of domains.
NRPTKeyPrefix = "NetBird-Match"
// NRPTKeyPrefix starts the name of every NRPT rule key this client creates:
// the match rules, the catch-all, and the .local exemption. Cleanup
// enumerates by this prefix, so a new kind of rule is removed by existing
// code as long as its key starts here.
NRPTKeyPrefix = "NetBird-"
// nrptMatchKeyName names the match-domain rules. Older versions used
// different layouts under the same name: a single unsuffixed key, then one
// key per domain, now one key per batch of domains.
nrptMatchKeyName = NRPTKeyPrefix + "Match"
// DNSPolicyConfigRoot holds the NRPT rules of the local policy store.
DNSPolicyConfigRoot = `SYSTEM\CurrentControlSet\Services\Dnscache\Parameters\DnsPolicyConfig`
@@ -53,8 +61,24 @@ const (
)
const (
dnsPolicyConfigMatchPath = DNSPolicyConfigRoot + `\` + NRPTKeyPrefix
gpoDnsPolicyConfigMatchPath = GPODNSPolicyConfigRoot + `\` + NRPTKeyPrefix
dnsPolicyConfigMatchPath = DNSPolicyConfigRoot + `\` + nrptMatchKeyName
gpoDnsPolicyConfigMatchPath = GPODNSPolicyConfigRoot + `\` + nrptMatchKeyName
dnsPolicyConfigExemptLocalPath = DNSPolicyConfigRoot + `\` + NRPTKeyPrefix + `ExemptLocal`
gpoDnsPolicyConfigExemptLocalPath = GPODNSPolicyConfigRoot + `\` + NRPTKeyPrefix + `ExemptLocal`
nrptCatchAllNamespace = "."
// nrptLocalNamespace is reserved for multicast DNS by RFC 6762: a unicast
// resolver must not answer for it. The catch-all rule would hand it to us
// anyway, so it gets an exemption rule of its own.
nrptLocalNamespace = ".local"
// envLegacyDNSResolution restores the pre-catch-all behaviour: the adapter's
// NameServer alone, leaving the OS free to query other adapters' resolvers in
// parallel. An escape hatch for setups that depend on a resolver of theirs
// still being reachable while connected, at the cost of the leak and of the
// race the catch-all rule exists to close.
envLegacyDNSResolution = "NB_USE_LEGACY_DNS_RESOLUTION"
dnsPolicyConfigVersionKey = "Version"
dnsPolicyConfigVersionValue = 2
@@ -100,19 +124,9 @@ func newHostManager(wgInterface WGIface) (*registryConfigurator, error) {
return nil, err
}
var useGPO bool
k, err := registry.OpenKey(registry.LOCAL_MACHINE, GPODNSPolicyConfigRoot, registry.QUERY_VALUE)
if err != nil {
log.Debugf("failed to open GPO DNS policy root: %v", err)
} else {
closer(k)
useGPO = true
log.Infof("detected GPO DNS policy configuration, using policy store")
}
configurator := &registryConfigurator{
guid: guid,
gpo: useGPO,
gpo: useGPOPolicyStore(),
}
origNameservers, err := configurator.captureOriginalNameservers()
@@ -293,6 +307,13 @@ func (r *registryConfigurator) disableWINSForInterface() error {
}
func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig, stateManager *statemanager.Manager) error {
// Clear every rule the previous apply installed before installing any new
// one, including a leftover catch-all: removal is unconditional so a rule
// from an earlier run cannot survive into a config that no longer wants it.
if err := r.removeDNSMatchPolicies(); err != nil {
log.Errorf("cleanup old dns match policies: %s", err)
}
if config.RouteAll {
if err := r.addDNSSetupForAll(config.ServerIP); err != nil {
return fmt.Errorf("add dns setup: %w", err)
@@ -318,8 +339,22 @@ func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig, stateManager
matchDomains = append(matchDomains, "."+strings.TrimSuffix(dConf.Domain, "."))
}
if err := r.removeDNSMatchPolicies(); err != nil {
log.Errorf("cleanup old dns match policies: %s", err)
// The root namespace is a match domain like any other: it just happens to
// match every name. Without it the adapter's NameServer only adds one more
// resolver to the set Windows queries in parallel, keeping whichever answer
// comes back first — which leaks every query to the local network and lets a
// resolver other than ours answer for a name we are authoritative for.
if config.RouteAll {
if parseBoolEnv(envLegacyDNSResolution) {
log.Infof("%s is set, leaving DNS resolution shared with the other adapters' resolvers instead of forcing it through %s", envLegacyDNSResolution, config.ServerIP)
} else {
matchDomains = append(matchDomains, nrptCatchAllNamespace)
log.Infof("routing every namespace through %s: DNS resolution is now exclusive to NetBird", config.ServerIP)
if err := r.addDNSExemptLocalPolicy(); err != nil {
return fmt.Errorf("add dns exempt policy: %w", err)
}
}
}
if len(matchDomains) != 0 {
@@ -397,6 +432,42 @@ func (r *registryConfigurator) addDNSMatchPolicy(domains []string, ip netip.Addr
return nil
}
// addDNSExemptLocalPolicy carves .local back out of the catch-all. RFC 6762
// reserves it for multicast DNS, so forwarding those names to a unicast
// upstream answers NXDOMAIN for hosts that do exist - printers, NAS boxes, and
// anything else announcing itself on the link - and the answer is authoritative
// enough that Windows stops looking. A rule naming the namespace with no
// servers hands it back to the DNS client untouched. A more specific rule still
// wins, so a match domain under .local keeps going through us.
func (r *registryConfigurator) addDNSExemptLocalPolicy() error {
var noServers netip.Addr
if err := r.configureDNSPolicy(dnsPolicyConfigExemptLocalPath, []string{nrptLocalNamespace}, noServers); err != nil {
return fmt.Errorf("configure exempt policy for %s: %w", nrptLocalNamespace, err)
}
if r.gpo {
if err := r.configureDNSPolicy(gpoDnsPolicyConfigExemptLocalPath, []string{nrptLocalNamespace}, noServers); err != nil {
return fmt.Errorf("configure gpo exempt policy for %s: %w", nrptLocalNamespace, err)
}
if err := refreshGroupPolicy(); err != nil {
log.Warnf("failed to refresh group policy: %v", err)
}
}
log.Infof("added NRPT exemption for %s, leaving it to the OS resolver", nrptLocalNamespace)
return nil
}
// configureDNSPolicy writes one NRPT rule. An invalid ip writes an exemption
// rule: the namespace with an empty server list, which tells the DNS client to
// resolve those names the way it would without any rule at all.
//
// The empty string is the whole difference, and it has to be written: dropping
// the value and clearing ConfigOptions instead produces a rule Windows treats
// as a no-op, keeps out of Get-DnsClientNrptPolicy -Effective, and ignores in
// favour of the catch-all. 0x8 says the server list is the meaningful part of
// the rule, and an empty list then means "no server, resolve normally".
func (r *registryConfigurator) configureDNSPolicy(policyPath string, domains []string, ip netip.Addr) error {
if err := removeRegistryKeyFromDNSPolicyConfig(policyPath); err != nil {
return fmt.Errorf("remove existing dns policy: %w", err)
@@ -416,7 +487,11 @@ func (r *registryConfigurator) configureDNSPolicy(policyPath string, domains []s
return fmt.Errorf("set %s: %w", dnsPolicyConfigNameKey, err)
}
if err := regKey.SetStringValue(dnsPolicyConfigGenericDNSServersKey, ip.String()); err != nil {
var servers string
if ip.IsValid() {
servers = ip.String()
}
if err := regKey.SetStringValue(dnsPolicyConfigGenericDNSServersKey, servers); err != nil {
return fmt.Errorf("set %s: %w", dnsPolicyConfigGenericDNSServersKey, err)
}
@@ -491,14 +566,22 @@ func (r *registryConfigurator) setInterfaceRegistryKeyStringValue(key, value str
return nil
}
// deleteInterfaceRegistryKeyProperty removes a value from the interface key.
// A value that is already gone, or an interface key that is, is not an error:
// the caller asked for the value not to be there, and a cleanup that runs twice
// has to reach its later steps on the second run as well.
func (r *registryConfigurator) deleteInterfaceRegistryKeyProperty(propertyKey string) error {
regKey, err := r.getInterfaceRegistryKey()
if err != nil {
switch {
case errors.Is(err, registry.ErrNotExist), errors.Is(err, syscall.ERROR_PATH_NOT_FOUND):
log.Debugf("interface key of %s does not exist, nothing to delete %s from", r.guid, propertyKey)
return nil
case err != nil:
return fmt.Errorf("get interface registry key: %w", err)
}
defer closer(regKey)
if err := regKey.DeleteValue(propertyKey); err != nil {
if err := regKey.DeleteValue(propertyKey); err != nil && !errors.Is(err, registry.ErrNotExist) {
return fmt.Errorf("delete registry key %s: %w", propertyKey, err)
}
return nil
@@ -514,8 +597,11 @@ func (r *registryConfigurator) getInterfaceRegistryKey() (registry.Key, error) {
}
func (r *registryConfigurator) restoreHostDNS() error {
// Propagated, unlike in applyDNSConfig: there we are about to write fresh
// rules over whatever survived, here we are leaving, and a rule left behind
// keeps sending every query to an address that is about to disappear.
if err := r.removeDNSMatchPolicies(); err != nil {
log.Errorf("remove dns match policies: %s", err)
return fmt.Errorf("remove dns match policies: %w", err)
}
if err := r.deleteInterfaceRegistryKeyProperty(interfaceConfigSearchListKey); err != nil {
@@ -524,7 +610,12 @@ func (r *registryConfigurator) restoreHostDNS() error {
go r.flushDNSCache()
return nil
// Last, and only on the way out, once no rule of ours is left: during a
// session the store is where the rules of this run live, and emptying it
// mid-session would have the next rule recreate it anyway. Propagated so a
// failure keeps the shutdown state for the next run to retry, rather than
// leaving the store to hold up every rule change from here on.
return removeEmptyGPOPolicyStore()
}
// removeDNSMatchPolicies deletes every NRPT rule this client may have created,
@@ -563,6 +654,73 @@ func (r *registryConfigurator) restoreUncleanShutdownDNS() error {
return r.restoreHostDNS()
}
// useGPOPolicyStore reports whether NRPT rules have to go into the group policy
// store, and clears an empty one out of the way first.
//
// The order is the point. A store left empty by an earlier run would otherwise
// decide this run too, sending its rules somewhere the resolver only reads when
// the policy engine next applies DNS client policy. Removing it before the
// choice is made leaves the local store authoritative for the whole session,
// including the first one after an upgrade.
func useGPOPolicyStore() bool {
if err := removeEmptyGPOPolicyStore(); err != nil {
// Nothing to retry against here: the worst case is the run going
// through the group policy store, which is where it would have gone
// before this check existed.
log.Warnf("%v", err)
}
k, err := registry.OpenKey(registry.LOCAL_MACHINE, GPODNSPolicyConfigRoot, registry.QUERY_VALUE)
if err != nil {
log.Debugf("failed to open GPO DNS policy root: %v", err)
return false
}
closer(k)
log.Infof("detected GPO DNS policy configuration, using policy store")
return true
}
// removeEmptyGPOPolicyStore deletes the group policy DnsPolicyConfig key once
// nothing is left in it. The key survives the deletion of the last rule it
// held, and the client treats its presence as "group policy configures the
// NRPT", so an empty one left behind keeps every later run writing rules there.
// Rules in that store reach the resolver only when the policy engine next
// applies DNS client policy, and a rule this client writes belongs to no GPO,
// so nothing schedules that application: both adding and removing a rule are
// held up by a minute or more, and for a removal that is a catch-all rule
// resolving every name over an interface that no longer exists. With the store
// absent the local one is authoritative and a change applies at once.
//
// A store that still holds rules, values or subkeys of somebody else's is left
// alone.
func removeEmptyGPOPolicyStore() error {
k, err := registry.OpenKey(registry.LOCAL_MACHINE, GPODNSPolicyConfigRoot, registry.QUERY_VALUE)
switch {
case errors.Is(err, registry.ErrNotExist), errors.Is(err, syscall.ERROR_PATH_NOT_FOUND):
return nil
case err != nil:
return fmt.Errorf("open HKEY_LOCAL_MACHINE\\%s: %w", GPODNSPolicyConfigRoot, err)
}
info, err := k.Stat()
closer(k)
if err != nil {
return fmt.Errorf("stat HKEY_LOCAL_MACHINE\\%s: %w", GPODNSPolicyConfigRoot, err)
}
if info.SubKeyCount != 0 || info.ValueCount != 0 {
return nil
}
if err := registry.DeleteKey(registry.LOCAL_MACHINE, GPODNSPolicyConfigRoot); err != nil {
return fmt.Errorf("delete empty HKEY_LOCAL_MACHINE\\%s: %w", GPODNSPolicyConfigRoot, err)
}
log.Infof("removed the empty GPO DNS policy store, leaving the local one authoritative")
return nil
}
// listNRPTRuleKeys returns the names of our NRPT rule keys under a policy store
// root. An absent root holds nothing to clean up, which is the normal state of
// the GPO store on a machine without DNS Client policy.
@@ -598,9 +756,17 @@ func listNRPTRuleKeys(root string) ([]string, error) {
func removeRegistryKeyFromDNSPolicyConfig(regKeyPath string) error {
k, err := registry.OpenKey(registry.LOCAL_MACHINE, regKeyPath, registry.QUERY_VALUE)
if err != nil {
log.Debugf("failed to open HKEY_LOCAL_MACHINE\\%s: %v", regKeyPath, err)
switch {
case errors.Is(err, registry.ErrNotExist), errors.Is(err, syscall.ERROR_PATH_NOT_FOUND):
// nothing to remove, which is the normal case for a rule this config
// never installed
log.Debugf("HKEY_LOCAL_MACHINE\\%s does not exist", regKeyPath)
return nil
case err != nil:
// anything else has to reach the caller: reporting success here would
// leave the rule in force while claiming it was removed, which is how a
// stale rule outlives the interface it points at
return fmt.Errorf("open HKEY_LOCAL_MACHINE\\%s: %w", regKeyPath, err)
}
closer(k)
@@ -636,6 +802,20 @@ func refreshGroupPolicy() error {
return nil
}
func parseBoolEnv(key string) bool {
val := os.Getenv(key)
if val == "" {
return false
}
parsed, err := strconv.ParseBool(val)
if err != nil {
log.Warnf("failed to parse %s=%q: %v", key, val, err)
return false
}
return parsed
}
func closer(closer io.Closer) {
if err := closer.Close(); err != nil {
log.Errorf("failed to close: %s", err)
+268
View File
@@ -8,6 +8,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/windows/registry"
"github.com/netbirdio/netbird/client/internal/winregistry"
)
// TestNRPTEntriesCleanupOnConfigChange tests that old NRPT entries are properly cleaned up
@@ -94,6 +96,145 @@ func TestNRPTEntriesCleanupOnConfigChange(t *testing.T) {
assert.False(t, exists, "NRPT rule 2 should NOT exist after reducing to 75 domains")
}
// TestNRPTCatchAllRule verifies that RouteAll adds the root namespace to the
// match rule instead of a rule of its own, that .local is carved back out with
// an empty server list, and that both go away when RouteAll is cleared or the
// host DNS is restored.
func TestNRPTCatchAllRule(t *testing.T) {
if testing.Short() {
t.Skip("skipping registry integration test in short mode")
}
defer cleanupRegistryKeys(t)
cleanupRegistryKeys(t)
testIP := netip.MustParseAddr("100.64.0.1")
testGUID := "{12345678-1234-1234-1234-123456789ABC}"
interfacePath := InterfaceConfigPath + `\` + testGUID
testKey, _, err := registry.CreateKey(registry.LOCAL_MACHINE, interfacePath, registry.SET_VALUE)
require.NoError(t, err, "Should create test interface registry key")
require.NoError(t, testKey.Close(), "close test interface registry key")
defer func() {
assert.NoError(t, registry.DeleteKey(registry.LOCAL_MACHINE, interfacePath), "delete test interface registry key")
}()
cfg := &registryConfigurator{guid: testGUID}
matchOnly := HostDNSConfig{
ServerIP: testIP,
Domains: []DomainConfig{{Domain: "example.com", MatchOnly: true}},
}
primary := HostDNSConfig{
ServerIP: testIP,
RouteAll: true,
Domains: []DomainConfig{{Domain: "example.com", MatchOnly: true}},
}
firstRule := fmt.Sprintf("%s-0", dnsPolicyConfigMatchPath)
// The root namespace is not a rule of its own: it rides in the match rule,
// which is the point of it not being a special case.
require.NoError(t, cfg.applyDNSConfig(matchOnly, nil))
names := ruleNamespaces(t, firstRule)
assert.Contains(t, names, ".example.com")
assert.NotContains(t, names, nrptCatchAllNamespace, "a match-only config must not claim every namespace")
require.NoError(t, cfg.applyDNSConfig(primary, nil))
names = ruleNamespaces(t, firstRule)
assert.Contains(t, names, ".example.com")
assert.Contains(t, names, nrptCatchAllNamespace, "RouteAll should add the root namespace to the match rule")
k, err := registry.OpenKey(registry.LOCAL_MACHINE, firstRule, registry.QUERY_VALUE)
require.NoError(t, err)
servers, _, err := k.GetStringValue(dnsPolicyConfigGenericDNSServersKey)
require.NoError(t, err)
assert.Equal(t, testIP.String(), servers, "every namespace in the rule resolves through our resolver")
require.NoError(t, k.Close(), "close match rule key")
// .local is carved back out: RFC 6762 reserves it for mDNS, so it needs a
// rule of its own — it is the one rule with a different server list.
ek, err := registry.OpenKey(registry.LOCAL_MACHINE, dnsPolicyConfigExemptLocalPath, registry.QUERY_VALUE)
require.NoError(t, err, "exemption rule should exist once the root namespace is claimed")
exemptNames, _, err := ek.GetStringsValue(dnsPolicyConfigNameKey)
require.NoError(t, err)
assert.Equal(t, []string{nrptLocalNamespace}, exemptNames, "the exemption should name only the mDNS namespace")
exemptServers, _, err := ek.GetStringValue(dnsPolicyConfigGenericDNSServersKey)
require.NoError(t, err, "the value has to be present, empty: without it Windows drops the rule")
assert.Empty(t, exemptServers, "an exemption rule lists no servers")
exemptOpts, _, err := ek.GetIntegerValue(dnsPolicyConfigConfigOptionsKey)
require.NoError(t, err)
assert.EqualValues(t, dnsPolicyConfigConfigOptionsValue, exemptOpts, "same options as a normal rule; the empty server list is what makes it an exemption")
require.NoError(t, ek.Close(), "close exemption rule key")
require.NoError(t, cfg.applyDNSConfig(matchOnly, nil))
names = ruleNamespaces(t, firstRule)
assert.NotContains(t, names, nrptCatchAllNamespace, "clearing RouteAll should drop the root namespace")
exists, err := registryKeyExists(dnsPolicyConfigExemptLocalPath)
require.NoError(t, err)
assert.False(t, exists, "exemption rule should go with the namespace it carves out of")
require.NoError(t, cfg.applyDNSConfig(primary, nil))
require.NoError(t, cfg.restoreHostDNS())
exists, err = registryKeyExists(firstRule)
require.NoError(t, err)
assert.False(t, exists, "restore should leave no rule behind")
}
// ruleNamespaces returns the namespaces an NRPT rule key claims.
func ruleNamespaces(t *testing.T, path string) []string {
t.Helper()
k, err := registry.OpenKey(registry.LOCAL_MACHINE, path, registry.QUERY_VALUE)
require.NoError(t, err, "rule key %s should exist", path)
defer k.Close()
names, _, err := k.GetStringsValue(dnsPolicyConfigNameKey)
require.NoError(t, err)
return names
}
// TestNRPTCatchAllRuleLegacyEnv verifies that NB_USE_LEGACY_DNS_RESOLUTION
// leaves the root namespace unclaimed, so no rule is written for a RouteAll
// config that carries no match domains.
func TestNRPTCatchAllRuleLegacyEnv(t *testing.T) {
if testing.Short() {
t.Skip("skipping registry integration test in short mode")
}
defer cleanupRegistryKeys(t)
cleanupRegistryKeys(t)
t.Setenv(envLegacyDNSResolution, "true")
testGUID := "{12345678-1234-1234-1234-123456789ABC}"
interfacePath := InterfaceConfigPath + `\` + testGUID
testKey, _, err := registry.CreateKey(registry.LOCAL_MACHINE, interfacePath, registry.SET_VALUE)
require.NoError(t, err, "Should create test interface registry key")
require.NoError(t, testKey.Close(), "close test interface registry key")
defer func() {
assert.NoError(t, registry.DeleteKey(registry.LOCAL_MACHINE, interfacePath), "delete test interface registry key")
}()
cfg := &registryConfigurator{guid: testGUID}
config := HostDNSConfig{
ServerIP: netip.MustParseAddr("100.64.0.1"),
RouteAll: true,
}
require.NoError(t, cfg.applyDNSConfig(config, nil))
// RouteAll with no match domains and the switch set leaves nothing to write.
exists, err := registryKeyExists(fmt.Sprintf("%s-0", dnsPolicyConfigMatchPath))
require.NoError(t, err)
assert.False(t, exists, "no rule should be written when the legacy env var is set")
exists, err = registryKeyExists(dnsPolicyConfigExemptLocalPath)
require.NoError(t, err)
assert.False(t, exists, "no exemption without a claimed root namespace")
}
func registryKeyExists(path string) (bool, error) {
k, err := registry.OpenKey(registry.LOCAL_MACHINE, path, registry.QUERY_VALUE)
if err != nil {
@@ -266,3 +407,130 @@ func TestNRPTDomainBatching(t *testing.T) {
})
}
}
// TestRemoveEmptyGPOPolicyStore verifies that cleanup takes the GPO policy
// store itself with it once our rules are gone, since the store existing keeps
// the local one from being applied, and that a store with somebody else's rule
// in it is left alone.
func TestRemoveEmptyGPOPolicyStore(t *testing.T) {
if testing.Short() {
t.Skip("skipping registry integration test in short mode")
}
t.Cleanup(func() { cleanupRegistryKeys(t) })
cleanupRegistryKeys(t)
testIP := netip.MustParseAddr("100.64.0.1")
cfg := &registryConfigurator{gpo: true}
// a store holding a rule of ours is kept, because the rule is still applied
require.NoError(t, cfg.addDNSMatchPolicy([]string{".example.com"}, testIP))
exists, err := registryKeyExists(gpoDnsPolicyConfigMatchPath + "-0")
require.NoError(t, err)
require.True(t, exists, "Should write the rule to the GPO policy store")
require.NoError(t, removeEmptyGPOPolicyStore())
exists, err = registryKeyExists(GPODNSPolicyConfigRoot)
require.NoError(t, err)
assert.True(t, exists, "Should keep a policy store that still holds a rule")
// once the rules are gone the store goes with them
require.NoError(t, cfg.removeDNSMatchPolicies())
require.NoError(t, removeEmptyGPOPolicyStore())
exists, err = registryKeyExists(GPODNSPolicyConfigRoot)
require.NoError(t, err)
assert.False(t, exists, "Should remove the GPO policy store once it is empty")
// A store is not ours to remove while somebody else has a rule in it. The
// rule is written volatile like our own: the rules above created the parent
// chain volatile, and Windows refuses a stable subkey under a volatile
// parent.
foreignRule := GPODNSPolicyConfigRoot + `\{2A3B4C5D-6E7F-4041-8283-84858687888A}`
foreignKey, _, err := winregistry.CreateVolatileKey(registry.LOCAL_MACHINE, foreignRule, registry.SET_VALUE)
require.NoError(t, err, "Should create a foreign GPO rule")
foreignKey.Close()
t.Cleanup(func() {
_ = registry.DeleteKey(registry.LOCAL_MACHINE, foreignRule)
_ = registry.DeleteKey(registry.LOCAL_MACHINE, GPODNSPolicyConfigRoot)
})
require.NoError(t, cfg.removeDNSMatchPolicies())
require.NoError(t, removeEmptyGPOPolicyStore())
exists, err = registryKeyExists(foreignRule)
require.NoError(t, err)
assert.True(t, exists, "Should not remove a foreign rule")
exists, err = registryKeyExists(GPODNSPolicyConfigRoot)
require.NoError(t, err)
assert.True(t, exists, "Should keep a policy store that still holds a foreign rule")
}
// TestDeleteInterfaceRegistryKeyPropertyTwice verifies that removing a value
// that is already gone, or one on an interface key that is, reports success.
// Teardown runs again after a failed cleanup, and the steps that follow this
// one have to be reached on that second run.
func TestDeleteInterfaceRegistryKeyPropertyTwice(t *testing.T) {
if testing.Short() {
t.Skip("skipping registry integration test in short mode")
}
testGUID := "{12345678-1234-1234-1234-123456789ABC}"
interfacePath := InterfaceConfigPath + `\` + testGUID
testKey, _, err := registry.CreateKey(registry.LOCAL_MACHINE, interfacePath, registry.SET_VALUE)
require.NoError(t, err, "Should create test interface registry key")
testKey.Close()
t.Cleanup(func() {
_ = registry.DeleteKey(registry.LOCAL_MACHINE, interfacePath)
})
cfg := &registryConfigurator{guid: testGUID}
require.NoError(t, cfg.setInterfaceRegistryKeyStringValue(interfaceConfigSearchListKey, "example.com"))
require.NoError(t, cfg.deleteInterfaceRegistryKeyProperty(interfaceConfigSearchListKey))
assert.NoError(t, cfg.deleteInterfaceRegistryKeyProperty(interfaceConfigSearchListKey),
"Should report success for a value that is already gone")
// and with the interface key itself gone, as it is once the adapter is
require.NoError(t, registry.DeleteKey(registry.LOCAL_MACHINE, interfacePath))
assert.NoError(t, cfg.deleteInterfaceRegistryKeyProperty(interfaceConfigSearchListKey),
"Should report success when the interface key does not exist")
}
// TestUseGPOPolicyStoreClearsEmptyStore verifies that the store is cleared
// before it is consulted, so an empty one left by an earlier run does not send
// this run's rules to the group policy store. A store somebody else has a rule
// in still decides where the rules go.
func TestUseGPOPolicyStoreClearsEmptyStore(t *testing.T) {
if testing.Short() {
t.Skip("skipping registry integration test in short mode")
}
t.Cleanup(func() { cleanupRegistryKeys(t) })
cleanupRegistryKeys(t)
// the leftover an earlier run used to keep, which the client read as
// "group policy configures the NRPT" for every run after it
emptyStore, _, err := winregistry.CreateVolatileKey(registry.LOCAL_MACHINE, GPODNSPolicyConfigRoot, registry.SET_VALUE)
require.NoError(t, err, "Should create the GPO policy store")
emptyStore.Close()
assert.False(t, useGPOPolicyStore(), "An empty store should not decide where the rules go")
exists, err := registryKeyExists(GPODNSPolicyConfigRoot)
require.NoError(t, err)
assert.False(t, exists, "Should clear the empty store before consulting it")
foreignRule := GPODNSPolicyConfigRoot + `\{2A3B4C5D-6E7F-4041-8283-84858687888A}`
foreignKey, _, err := winregistry.CreateVolatileKey(registry.LOCAL_MACHINE, foreignRule, registry.SET_VALUE)
require.NoError(t, err, "Should create a foreign GPO rule")
foreignKey.Close()
t.Cleanup(func() {
_ = registry.DeleteKey(registry.LOCAL_MACHINE, foreignRule)
_ = registry.DeleteKey(registry.LOCAL_MACHINE, GPODNSPolicyConfigRoot)
})
assert.True(t, useGPOPolicyStore(), "A store holding a rule should decide where the rules go")
exists, err = registryKeyExists(GPODNSPolicyConfigRoot)
require.NoError(t, err)
assert.True(t, exists, "Should keep a store that holds a rule")
}
+7 -10
View File
@@ -9,9 +9,9 @@ import (
"os"
"testing"
"go.uber.org/mock/gomock"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
"go.uber.org/mock/gomock"
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
"github.com/netbirdio/netbird/client/iface"
@@ -24,6 +24,10 @@ import (
nbdns "github.com/netbirdio/netbird/dns"
)
// testIFaceBlackList mirrors the overlay prefixes profilemanager.DefaultInterfaceBlacklist
// carries. Declared here rather than imported because profilemanager imports this package.
var testIFaceBlackList = []string{"wt", "utun", "tun0"}
func TestUpdateDNSServer(t *testing.T) {
nameServers := []nbdns.NameServer{
@@ -243,10 +247,7 @@ func TestUpdateDNSServer(t *testing.T) {
for n, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
privKey, _ := wgtypes.GenerateKey()
newNet, err := stdnet.NewNet(context.Background(), nil)
if err != nil {
t.Fatal(err)
}
newNet := stdnet.NewNet(context.Background(), testIFaceBlackList)
opts := iface.WGIFaceOpts{
IFaceName: fmt.Sprintf("utun230%d", n),
@@ -348,11 +349,7 @@ func TestDNSFakeResolverHandleUpdates(t *testing.T) {
defer t.Setenv("NB_WG_KERNEL_DISABLED", ov)
t.Setenv("NB_WG_KERNEL_DISABLED", "true")
newNet, err := stdnet.NewNet(context.Background(), []string{"utun2301"})
if err != nil {
t.Errorf("create stdnet: %v", err)
return
}
newNet := stdnet.NewNet(context.Background(), []string{"utun2301"})
privKey, _ := wgtypes.GeneratePrivateKey()
opts := iface.WGIFaceOpts{
+2 -6
View File
@@ -394,11 +394,7 @@ func createWgInterfaceWithBind(t *testing.T) (*iface.WGIface, error) {
defer t.Setenv("NB_WG_KERNEL_DISABLED", ov)
t.Setenv("NB_WG_KERNEL_DISABLED", "true")
newNet, err := stdnet.NewNet(context.Background(), []string{"utun2301"})
if err != nil {
t.Fatalf("create stdnet: %v", err)
return nil, err
}
newNet := stdnet.NewNet(context.Background(), []string{"utun2301"})
privKey, _ := wgtypes.GeneratePrivateKey()
@@ -423,7 +419,7 @@ func createWgInterfaceWithBind(t *testing.T) (*iface.WGIface, error) {
return nil, err
}
pf, err := uspfilter.Create(wgIface, false, flowLogger, iface.DefaultMTU)
pf, err := uspfilter.Create(uspfilter.Config{IFace: wgIface, FlowLogger: flowLogger, MTU: iface.DefaultMTU})
if err != nil {
t.Fatalf("failed to create uspfilter: %v", err)
return nil, err
+136 -87
View File
@@ -6,6 +6,7 @@ import (
"net"
"net/netip"
"runtime"
"slices"
"strconv"
"sync"
"time"
@@ -17,17 +18,20 @@ import (
nberrors "github.com/netbirdio/netbird/client/errors"
firewall "github.com/netbirdio/netbird/client/firewall/manager"
"github.com/netbirdio/netbird/client/internal/ebpf"
ebpfMgr "github.com/netbirdio/netbird/client/internal/ebpf/manager"
)
const (
customPort = 5053
// randomPortAttempts bounds the search for a port free on both protocols.
randomPortAttempts = 5
)
var (
defaultIP = netip.MustParseAddr("127.0.0.1")
customIP = netip.MustParseAddr("127.0.0.153")
// dnatProtocols are the protocols the port 53 redirect covers.
dnatProtocols = []firewall.Protocol{firewall.ProtocolUDP, firewall.ProtocolTCP}
)
type serviceViaListener struct {
@@ -40,9 +44,20 @@ type serviceViaListener struct {
listenPort uint16
listenerIsRunning bool
listenerFlagLock sync.Mutex
ebpfService ebpfMgr.Manager
firewall Firewall
tcpDNATConfigured bool
// dnatRules holds the port 53 redirects that are installed and not yet
// removed, so a removal that fails can be retried.
dnatRules []dnatRule
}
// dnatRule is a port 53 redirect as it was installed. The target is kept with
// the rule because the listener can come back on a different address or port,
// and a retried removal has to name the address and port the rule was added
// with, not the ones in use now.
type dnatRule struct {
protocol firewall.Protocol
ip netip.Addr
port uint16
}
func newServiceViaListener(wgIface WGIface, customAddr *netip.AddrPort, fw Firewall) *serviceViaListener {
@@ -112,34 +127,93 @@ func (s *serviceViaListener) Listen() error {
}
}()
// When eBPF redirects UDP port 53 to our listen port, TCP still needs
// a DNAT rule because eBPF only handles UDP.
if s.ebpfService != nil && s.firewall != nil && s.listenPort != DefaultPort {
if err := s.firewall.AddOutputDNAT(s.listenIP, firewall.ProtocolTCP, DefaultPort, s.listenPort); err != nil {
log.Warnf("failed to add DNS TCP DNAT rule, TCP DNS on port 53 will not work: %v", err)
} else {
s.tcpDNATConfigured = true
log.Infof("added DNS TCP DNAT rule: %s:%d -> %s:%d", s.listenIP, DefaultPort, s.listenIP, s.listenPort)
}
if s.listenPort != DefaultPort {
s.setupDNAT()
}
return nil
}
// setupDNAT redirects port 53 to the port the DNS server actually listens on.
// Both protocols must be redirected or none: RuntimePort reports port 53 only
// while the full redirect is in place, so a half-configured redirect would
// advertise a resolver that answers over one protocol.
func (s *serviceViaListener) setupDNAT() {
if s.firewall == nil {
log.Errorf("no firewall manager available to redirect DNS port %d to %d, "+
"clients pointed at %s will not reach the resolver", DefaultPort, s.listenPort, s.listenIP)
return
}
// Clear whatever an earlier removal left behind first. Those rules can point
// at an address or port this listener no longer uses, and they are matched
// before anything added now, so adding a redirect on top of one would keep
// sending port 53 traffic to the previous listener while reporting the
// redirect as complete. The rules stay recorded for a later attempt.
if err := s.removeDNAT(); err != nil {
log.Errorf("failed to remove stale DNS DNAT rules, leaving port %d redirected to the previous listener: %v",
DefaultPort, err)
return
}
for _, proto := range dnatProtocols {
if err := s.firewall.AddOutputDNAT(s.listenIP, proto, DefaultPort, s.listenPort); err != nil {
log.Errorf("failed to add DNS %s DNAT rule, DNS on port %d will not work: %v",
proto, DefaultPort, err)
if err := s.removeDNAT(); err != nil {
log.Warnf("failed to roll back DNS DNAT rules, retrying on stop: %v", err)
}
return
}
s.dnatRules = append(s.dnatRules, dnatRule{protocol: proto, ip: s.listenIP, port: s.listenPort})
}
log.Infof("added DNS DNAT rules: %s:%d -> %s:%d (UDP + TCP)", s.listenIP, DefaultPort, s.listenIP, s.listenPort)
}
// removeDNAT removes every installed port 53 redirect. A rule whose removal
// fails stays recorded so a later setup or Stop retries it, rather than leaving
// port 53 pointing at a resolver that is no longer listening.
func (s *serviceViaListener) removeDNAT() error {
if s.firewall == nil {
return nil
}
var merr *multierror.Error
var remaining []dnatRule
for _, rule := range s.dnatRules {
if err := s.firewall.RemoveOutputDNAT(rule.ip, rule.protocol, DefaultPort, rule.port); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove DNS %s DNAT rule for %s:%d: %w",
rule.protocol, rule.ip, rule.port, err))
remaining = append(remaining, rule)
}
}
s.dnatRules = remaining
return nberrors.FormatErrorOrNil(merr)
}
func (s *serviceViaListener) Stop() error {
s.listenerFlagLock.Lock()
defer s.listenerFlagLock.Unlock()
var merr *multierror.Error
// Redirects are removed even when the listener is already stopped, so that
// a removal which failed earlier is retried instead of leaving port 53
// pointing at a resolver that no longer listens.
if err := s.removeDNAT(); err != nil {
merr = multierror.Append(merr, err)
}
if !s.listenerIsRunning {
return nil
return nberrors.FormatErrorOrNil(merr)
}
s.listenerIsRunning = false
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var merr *multierror.Error
if err := s.server.ShutdownContext(ctx); err != nil {
merr = multierror.Append(merr, fmt.Errorf("stop DNS UDP server: %w", err))
}
@@ -148,19 +222,6 @@ func (s *serviceViaListener) Stop() error {
merr = multierror.Append(merr, fmt.Errorf("stop DNS TCP server: %w", err))
}
if s.tcpDNATConfigured && s.firewall != nil {
if err := s.firewall.RemoveOutputDNAT(s.listenIP, firewall.ProtocolTCP, DefaultPort, s.listenPort); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove DNS TCP DNAT rule: %w", err))
}
s.tcpDNATConfigured = false
}
if s.ebpfService != nil {
if err := s.ebpfService.FreeDNSFwd(); err != nil {
merr = multierror.Append(merr, fmt.Errorf("stop traffic forwarder: %w", err))
}
}
return nberrors.FormatErrorOrNil(merr)
}
@@ -177,11 +238,23 @@ func (s *serviceViaListener) RuntimePort() int {
s.listenerFlagLock.Lock()
defer s.listenerFlagLock.Unlock()
if s.ebpfService != nil {
if s.redirectInstalled() {
return DefaultPort
} else {
return int(s.listenPort)
}
return int(s.listenPort)
}
// redirectInstalled reports whether every protocol is redirected from port 53
// to the address and port the listener currently serves. Rules left over from
// an earlier listener do not count.
func (s *serviceViaListener) redirectInstalled() bool {
for _, proto := range dnatProtocols {
current := dnatRule{protocol: proto, ip: s.listenIP, port: s.listenPort}
if !slices.Contains(s.dnatRules, current) {
return false
}
}
return true
}
func (s *serviceViaListener) RuntimeIP() netip.Addr {
@@ -190,30 +263,29 @@ func (s *serviceViaListener) RuntimeIP() netip.Addr {
// evalListenAddress figures out the listen address for the DNS server.
// IPv4-only: all peers have a v4 overlay address, and DNS config points to v4.
// First checks port 53 on WG interface or lo, then tries eBPF on a random port,
// then falls back to port 5053.
// Prefers port 53 on the overlay interface or lo, so no redirect is needed at
// all; when it is taken it falls back to port 5053 and then to a random free
// port, both of which need the port 53 redirect set up by setupDNAT.
func (s *serviceViaListener) evalListenAddress() (netip.Addr, uint16, error) {
if s.customAddr != nil {
return s.customAddr.Addr(), s.customAddr.Port(), nil
}
ip, ok := s.testFreePort(DefaultPort)
if ok {
if ip, ok := s.testFreePort(DefaultPort); ok {
return ip, DefaultPort, nil
}
ebpfSrv, port, ok := s.tryToUseeBPF()
if ok {
s.ebpfService = ebpfSrv
return s.wgInterface.Address().IP, port, nil
}
ip, ok = s.testFreePort(customPort)
if ok {
if ip, ok := s.testFreePort(customPort); ok {
return ip, customPort, nil
}
return netip.Addr{}, 0, fmt.Errorf("failed to find a free port for DNS server")
ip := s.wgInterface.Address().IP
port, err := s.randomFreePort(ip)
if err != nil {
return netip.Addr{}, 0, fmt.Errorf("find a free port for DNS server: %w", err)
}
return ip, port, nil
}
func (s *serviceViaListener) testFreePort(port int) (netip.Addr, bool) {
@@ -260,48 +332,25 @@ func (s *serviceViaListener) tryToBind(ip netip.Addr, port int) bool {
return true
}
// tryToUseeBPF decides whether to apply eBPF program to capture DNS traffic on port 53.
// This is needed because on some operating systems if we start a DNS server not on a default port 53,
// the domain name resolution won't work. So, in case we are running on Linux and picked a free
// port we should fall back to the eBPF solution that will capture traffic on port 53 and forward
// it to a local DNS server running on the chosen port.
func (s *serviceViaListener) tryToUseeBPF() (ebpfMgr.Manager, uint16, bool) {
if runtime.GOOS != "linux" {
return nil, 0, false
// randomFreePort returns a port that is free on ip for both UDP and TCP, since
// the DNS server binds both. The probe listeners are closed again, so the port
// is only likely, not guaranteed, to still be free when the server binds it.
func (s *serviceViaListener) randomFreePort(ip netip.Addr) (uint16, error) {
for range randomPortAttempts {
probeListener, err := net.ListenUDP("udp4", &net.UDPAddr{})
if err != nil {
return 0, fmt.Errorf("bind random port: %w", err)
}
port := uint16(probeListener.LocalAddr().(*net.UDPAddr).Port)
if err := probeListener.Close(); err != nil {
return 0, fmt.Errorf("free up probed port: %w", err)
}
if s.tryToBind(ip, int(port)) {
return port, nil
}
}
port, err := s.generateFreePort() //nolint:staticcheck,unused
if err != nil {
log.Warnf("failed to generate a free port for eBPF DNS forwarder server: %s", err)
return nil, 0, false
}
ebpfSrv := ebpf.GetEbpfManagerInstance()
err = ebpfSrv.LoadDNSFwd(s.wgInterface.Address().IP, int(port))
if err != nil {
log.Warnf("failed to load DNS forwarder eBPF program, error: %s", err)
return nil, 0, false
}
return ebpfSrv, port, true
}
func (s *serviceViaListener) generateFreePort() (uint16, error) {
ok := s.tryToBind(s.wgInterface.Address().IP, customPort)
if ok {
return customPort, nil
}
probeListener, err := net.ListenUDP("udp4", &net.UDPAddr{})
if err != nil {
log.Debugf("failed to bind random port for DNS: %s", err)
return 0, err
}
port := uint16(probeListener.LocalAddr().(*net.UDPAddr).Port)
if err = probeListener.Close(); err != nil {
log.Debugf("failed to free up DNS port: %s", err)
return 0, err
}
return port, nil
return 0, fmt.Errorf("no port free for UDP and TCP on %s after %d attempts", ip, randomPortAttempts)
}
@@ -1,6 +1,7 @@
package dns
import (
"errors"
"fmt"
"net"
"net/netip"
@@ -10,6 +11,8 @@ import (
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
firewall "github.com/netbirdio/netbird/client/firewall/manager"
)
func TestServiceViaListener_TCPAndUDP(t *testing.T) {
@@ -84,3 +87,133 @@ func TestServiceViaListener_TCPAndUDP(t *testing.T) {
require.NotEmpty(t, tcpResp.Answer)
assert.Contains(t, tcpResp.Answer[0].String(), "192.0.2.1", "TCP response should contain expected IP")
}
type dnatCall struct {
rule dnatRule
added bool
}
// fakeFirewall records DNAT calls and fails the ones named in addErrs/removeErrs.
type fakeFirewall struct {
calls []dnatCall
addErrs map[firewall.Protocol]error
removeErrs map[firewall.Protocol]error
}
func (f *fakeFirewall) AddOutputDNAT(ip netip.Addr, protocol firewall.Protocol, _, translatedPort uint16) error {
if err := f.addErrs[protocol]; err != nil {
return err
}
f.calls = append(f.calls, dnatCall{rule: dnatRule{protocol: protocol, ip: ip, port: translatedPort}, added: true})
return nil
}
func (f *fakeFirewall) RemoveOutputDNAT(ip netip.Addr, protocol firewall.Protocol, _, translatedPort uint16) error {
if err := f.removeErrs[protocol]; err != nil {
return err
}
f.calls = append(f.calls, dnatCall{rule: dnatRule{protocol: protocol, ip: ip, port: translatedPort}})
return nil
}
func newDNATTestService(fw Firewall) *serviceViaListener {
return &serviceViaListener{
listenIP: netip.MustParseAddr("100.64.0.1"),
listenPort: customPort,
firewall: fw,
}
}
func TestSetupDNAT_BothProtocols(t *testing.T) {
svc := newDNATTestService(&fakeFirewall{})
svc.setupDNAT()
assert.Len(t, svc.dnatRules, len(dnatProtocols))
assert.Equal(t, DefaultPort, svc.RuntimePort(), "port 53 is advertised once both redirects are installed")
}
func TestSetupDNAT_RollsBackPartialRedirect(t *testing.T) {
fw := &fakeFirewall{addErrs: map[firewall.Protocol]error{firewall.ProtocolTCP: errors.New("nftables busy")}}
svc := newDNATTestService(fw)
svc.setupDNAT()
assert.Empty(t, svc.dnatRules, "the UDP redirect installed before the failure must be rolled back")
assert.Equal(t, int(svc.listenPort), svc.RuntimePort(), "an incomplete redirect must not advertise port 53")
udp := dnatRule{protocol: firewall.ProtocolUDP, ip: svc.listenIP, port: svc.listenPort}
assert.Contains(t, fw.calls, dnatCall{rule: udp}, "UDP removal should have been attempted")
}
// A rollback that fails must keep the rule recorded, so port 53 is not left
// redirected to a resolver that no longer listens.
func TestStop_RetriesFailedDNATRemoval(t *testing.T) {
fw := &fakeFirewall{
addErrs: map[firewall.Protocol]error{firewall.ProtocolTCP: errors.New("nftables busy")},
removeErrs: map[firewall.Protocol]error{firewall.ProtocolUDP: errors.New("nftables busy")},
}
svc := newDNATTestService(fw)
svc.setupDNAT()
udp := dnatRule{protocol: firewall.ProtocolUDP, ip: svc.listenIP, port: svc.listenPort}
require.Equal(t, []dnatRule{udp}, svc.dnatRules, "a failed rollback keeps the rule for a later retry")
require.Error(t, svc.Stop(), "the failing removal should be reported")
require.Equal(t, []dnatRule{udp}, svc.dnatRules)
delete(fw.removeErrs, firewall.ProtocolUDP)
require.NoError(t, svc.Stop(), "a later stop retries the removal")
assert.Empty(t, svc.dnatRules)
}
// A stale rule that cannot be removed is matched before anything added now, so
// no new redirect may be installed on top of it and port 53 must not be
// advertised as reaching this listener.
func TestSetupDNAT_AbortsWhileStaleRuleRemains(t *testing.T) {
fw := &fakeFirewall{removeErrs: map[firewall.Protocol]error{firewall.ProtocolUDP: errors.New("nftables busy")}}
svc := newDNATTestService(fw)
stalePort := svc.listenPort
svc.setupDNAT()
require.Error(t, svc.Stop())
staleUDP := dnatRule{protocol: firewall.ProtocolUDP, ip: svc.listenIP, port: stalePort}
require.Equal(t, []dnatRule{staleUDP}, svc.dnatRules)
svc.listenPort = stalePort + 1
fw.calls = nil
svc.setupDNAT()
assert.Equal(t, []dnatRule{staleUDP}, svc.dnatRules, "the stale rule stays recorded for a later attempt")
for _, call := range fw.calls {
assert.False(t, call.added, "no redirect may be installed while a stale one is still in place")
}
assert.Equal(t, int(svc.listenPort), svc.RuntimePort(), "port 53 must not be advertised")
}
// A rule left behind by a failed removal must be removed with the address and
// port it was installed with, even when the listener has since moved to another
// port, and it must not count towards the redirect the new listener advertises.
func TestSetupDNAT_ClearsStaleRuleAfterPortChange(t *testing.T) {
fw := &fakeFirewall{removeErrs: map[firewall.Protocol]error{firewall.ProtocolUDP: errors.New("nftables busy")}}
svc := newDNATTestService(fw)
stalePort := svc.listenPort
svc.setupDNAT()
require.Error(t, svc.Stop())
staleUDP := dnatRule{protocol: firewall.ProtocolUDP, ip: svc.listenIP, port: stalePort}
require.Equal(t, []dnatRule{staleUDP}, svc.dnatRules)
delete(fw.removeErrs, firewall.ProtocolUDP)
svc.listenPort = stalePort + 1
fw.calls = nil
svc.setupDNAT()
assert.Contains(t, fw.calls, dnatCall{rule: staleUDP}, "the stale rule must be removed with its original port")
assert.Len(t, svc.dnatRules, len(dnatProtocols))
assert.Equal(t, DefaultPort, svc.RuntimePort(), "the new listener is fully redirected")
for _, rule := range svc.dnatRules {
assert.Equal(t, svc.listenPort, rule.port, "only rules for the current listener remain")
}
}
+111 -14
View File
@@ -54,12 +54,20 @@ type DNSForwarder struct {
ttl uint32
statusRecorder *peer.Status
dnsServer *dns.Server
mux *dns.ServeMux
tcpServer *dns.Server
tcpMux *dns.ServeMux
mux *dns.ServeMux
tcpMux *dns.ServeMux
mutex sync.RWMutex
mutex sync.RWMutex
// closed records that Close has run, so a Listen still in flight does not
// go on to serve sockets nobody will shut down.
closed bool
// The sockets are kept alongside the servers because closing them is the
// only stop that always works: a server whose ActivateAndServe has not run
// yet refuses to shut down, and would otherwise start serving afterwards.
udpConn net.PacketConn
tcpLn net.Listener
dnsServer *dns.Server
tcpServer *dns.Server
fwdEntries []*ForwarderEntry
firewall firewaller
resolver resolver
@@ -106,7 +114,7 @@ func (f *DNSForwarder) Listen(entries []*ForwarderEntry) error {
mux := dns.NewServeMux()
f.mux = mux
mux.HandleFunc(".", f.handleDNSQueryUDP)
f.dnsServer = &dns.Server{
dnsServer := &dns.Server{
PacketConn: udpLn,
Handler: mux,
}
@@ -114,22 +122,32 @@ func (f *DNSForwarder) Listen(entries []*ForwarderEntry) error {
tcpMux := dns.NewServeMux()
f.tcpMux = tcpMux
tcpMux.HandleFunc(".", f.handleDNSQueryTCP)
f.tcpServer = &dns.Server{
tcpServer := &dns.Server{
Listener: tcpLn,
Handler: tcpMux,
}
f.UpdateDomains(entries)
if !f.publish(udpLn, tcpLn, dnsServer, tcpServer, entries) {
log.Infof("DNS forwarder on %s was closed before it started serving", addrDesc)
if err := udpLn.Close(); err != nil {
log.Debugf("close UDP listener of a closed forwarder: %v", err)
}
if err := tcpLn.Close(); err != nil {
log.Debugf("close TCP listener of a closed forwarder: %v", err)
}
return nil
}
log.Debugf("DNS forwarder serving %d domains", len(entries))
errCh := make(chan error, 2)
go func() {
log.Infof("DNS UDP listener running on %s", addrDesc)
errCh <- f.dnsServer.ActivateAndServe()
errCh <- dnsServer.ActivateAndServe()
}()
go func() {
log.Infof("DNS TCP listener running on %s", addrDesc)
errCh <- f.tcpServer.ActivateAndServe()
errCh <- tcpServer.ActivateAndServe()
}()
return <-errCh
@@ -151,6 +169,46 @@ func (f *DNSForwarder) createTCPListener(netstackNet *netstack.Net) (net.Listene
return net.ListenTCP("tcp", net.TCPAddrFromAddrPort(f.listenAddress))
}
// publish hands the sockets, servers and entries to the forwarder so Close can
// reach them and Domains can report them, and says whether serving may begin.
// Listen runs on its own goroutine, so a Close can arrive before it gets this
// far; false means the caller must close what it created instead of serving on
// it.
//
// The entries go in under the same lock rather than afterwards. Anything that
// reads them in between would otherwise see a forwarder that is listening and
// serves no domain, which for a caller rebuilding one means it comes back
// refusing every routed query.
func (f *DNSForwarder) publish(
udpConn net.PacketConn,
tcpLn net.Listener,
dnsServer, tcpServer *dns.Server,
entries []*ForwarderEntry,
) bool {
f.mutex.Lock()
defer f.mutex.Unlock()
if f.closed {
return false
}
f.udpConn = udpConn
f.tcpLn = tcpLn
f.dnsServer = dnsServer
f.tcpServer = tcpServer
f.fwdEntries = entries
return true
}
// Domains returns the entries currently being served. The slice is replaced
// wholesale by UpdateDomains rather than mutated, so the caller may read it but
// must not write to it.
func (f *DNSForwarder) Domains() []*ForwarderEntry {
f.mutex.RLock()
defer f.mutex.RUnlock()
return f.fwdEntries
}
func (f *DNSForwarder) UpdateDomains(entries []*ForwarderEntry) {
f.mutex.Lock()
defer f.mutex.Unlock()
@@ -189,19 +247,45 @@ func (f *DNSForwarder) removeStaleCacheEntries(oldEntries, newEntries []*Forward
}
func (f *DNSForwarder) Close(ctx context.Context) error {
// Marked closed under the lock so a Listen that has not published its
// servers yet gives up instead of racing this shutdown. The shutdowns
// themselves block, so they run outside it.
f.mutex.Lock()
f.closed = true
dnsServer, tcpServer := f.dnsServer, f.tcpServer
udpConn, tcpLn := f.udpConn, f.tcpLn
f.mutex.Unlock()
var result *multierror.Error
if f.dnsServer != nil {
if err := f.dnsServer.ShutdownContext(ctx); err != nil {
if dnsServer != nil {
if err := shutdownServer(ctx, dnsServer); err != nil {
result = multierror.Append(result, fmt.Errorf("UDP shutdown: %w", err))
}
}
if f.tcpServer != nil {
if err := f.tcpServer.ShutdownContext(ctx); err != nil {
if tcpServer != nil {
if err := shutdownServer(ctx, tcpServer); err != nil {
result = multierror.Append(result, fmt.Errorf("TCP shutdown: %w", err))
}
}
// The sockets are closed even when the shutdowns above reported nothing to
// do. A server that has been published but has not reached
// ActivateAndServe refuses to shut down, and closing what it was about to
// serve on is what stops it: the alternative is a listener still answering
// on an interface that has gone away. A shutdown that did run has already
// closed these, so the second close is expected to fail.
if udpConn != nil {
if err := udpConn.Close(); err != nil {
log.Debugf("close UDP socket of the DNS forwarder: %v", err)
}
}
if tcpLn != nil {
if err := tcpLn.Close(); err != nil {
log.Debugf("close TCP socket of the DNS forwarder: %v", err)
}
}
return nberrors.FormatErrorOrNil(result)
}
@@ -514,3 +598,16 @@ func attachEDE(resp *dns.Msg, code uint16, text string) {
}
opt.Option = append(opt.Option, &dns.EDNS0_EDE{InfoCode: code, ExtraText: text})
}
// shutdownServer shuts a server down gracefully, treating "never started" as
// success. A server that was published but has not reached ActivateAndServe
// has nothing to wind down, and the caller closes its socket regardless, which
// is what actually stops it. dns exports no sentinel for this, so the message
// is all there is to match on.
func shutdownServer(ctx context.Context, server *dns.Server) error {
err := server.ShutdownContext(ctx)
if err == nil || strings.Contains(err.Error(), "server not started") {
return nil
}
return err
}
+52
View File
@@ -1238,3 +1238,55 @@ func TestDNSForwarder_EmptyQuery(t *testing.T) {
assert.Nil(t, mockWriter.GetLastResponse(), "Should not write response for empty query")
}
// TestDNSForwarder_ClosedBeforeItServes covers Listen reaching the point of
// serving after the forwarder has already been closed. Listen runs on its own
// goroutine, so it can get there late, and a socket it starts serving then is
// one nothing will ever close: on Android it keeps answering on an interface
// that has been replaced. The close is sequenced first here rather than raced,
// which pins the same state deterministically.
func TestDNSForwarder_ClosedBeforeItServes(t *testing.T) {
f := NewDNSForwarder(netip.MustParseAddrPort("127.0.0.1:0"), 60, nil, nil, nil)
require.NoError(t, f.Close(context.Background()), "closing a forwarder that never started")
done := make(chan error, 1)
go func() { done <- f.Listen(nil) }()
select {
case err := <-done:
assert.NoError(t, err, "a closed forwarder should give up quietly, not serve")
case <-time.After(5 * time.Second):
t.Fatal("Listen went on to serve after the forwarder was closed")
}
}
// TestDNSForwarder_CloseStopsUnactivatedServers covers the window between
// Listen publishing its servers and reaching ActivateAndServe. A server that
// has not been activated refuses to shut down, so Close has to close the
// sockets itself or they are left serving.
func TestDNSForwarder_CloseStopsUnactivatedServers(t *testing.T) {
f := NewDNSForwarder(netip.MustParseAddrPort("127.0.0.1:0"), 60, nil, nil, nil)
udpConn, err := f.createUDPListener(nil)
require.NoError(t, err, "create UDP listener")
tcpLn, err := f.createTCPListener(nil)
require.NoError(t, err, "create TCP listener")
// Published but deliberately never activated, which is the state Listen is
// in for the moment before it starts serving.
require.True(t, f.publish(udpConn, tcpLn, &dns.Server{PacketConn: udpConn}, &dns.Server{Listener: tcpLn}, nil),
"publishing to an open forwarder")
tcpAddr := tcpLn.Addr().String()
require.NoError(t, f.Close(context.Background()), "close should report no error for servers it could not shut down")
_, err = tcpLn.Accept()
assert.Error(t, err, "the TCP socket should be closed after Close")
conn, err := net.DialTimeout("tcp", tcpAddr, time.Second)
if err == nil {
_ = conn.Close()
t.Fatal("the forwarder is still accepting connections after Close")
}
}
+21 -7
View File
@@ -3,7 +3,6 @@ package dnsfwd
import (
"context"
"fmt"
"net"
"net/netip"
"os"
"strconv"
@@ -118,6 +117,16 @@ func (m *Manager) UpdateDomains(entries []*ForwarderEntry) {
m.dnsForwarder.UpdateDomains(entries)
}
// Domains returns the entries currently being served, or nil when the
// forwarder is not running.
func (m *Manager) Domains() []*ForwarderEntry {
if m.dnsForwarder == nil {
return nil
}
return m.dnsForwarder.Domains()
}
func (m *Manager) Stop(ctx context.Context) error {
if m.dnsForwarder == nil {
return nil
@@ -160,12 +169,13 @@ func (m *Manager) allowDNSFirewall() error {
return nil
}
dnsRules, err := m.firewall.AddPeerFiltering(nil, net.IP{0, 0, 0, 0}, firewall.ProtocolUDP, nil, dport, firewall.ActionAccept, "")
anyV4 := []netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)}
dnsRule, err := m.firewall.AddFilterRule(nil, anyV4, firewall.Network{}, firewall.ProtocolUDP, nil, dport, firewall.ActionAccept)
if err != nil {
return fmt.Errorf("add udp firewall rule: %w", err)
}
tcpRules, err := m.firewall.AddPeerFiltering(nil, net.IP{0, 0, 0, 0}, firewall.ProtocolTCP, nil, dport, firewall.ActionAccept, "")
tcpRule, err := m.firewall.AddFilterRule(nil, anyV4, firewall.Network{}, firewall.ProtocolTCP, nil, dport, firewall.ActionAccept)
if err != nil {
return fmt.Errorf("add tcp firewall rule: %w", err)
}
@@ -174,8 +184,12 @@ func (m *Manager) allowDNSFirewall() error {
return fmt.Errorf("flush: %w", err)
}
m.fwRules = dnsRules
m.tcpRules = tcpRules
if dnsRule != nil {
m.fwRules = []firewall.Rule{dnsRule}
}
if tcpRule != nil {
m.tcpRules = []firewall.Rule{tcpRule}
}
m.registerNetstackServices()
@@ -209,12 +223,12 @@ func (m *Manager) unregisterNetstackServices() {
func (m *Manager) dropDNSFirewall() error {
var mErr *multierror.Error
for _, rule := range m.fwRules {
if err := m.firewall.DeletePeerRule(rule); err != nil {
if err := m.firewall.DeleteFilterRule(rule); err != nil {
mErr = multierror.Append(mErr, fmt.Errorf("failed to delete DNS router rules, err: %v", err))
}
}
for _, rule := range m.tcpRules {
if err := m.firewall.DeletePeerRule(rule); err != nil {
if err := m.firewall.DeleteFilterRule(rule); err != nil {
mErr = multierror.Append(mErr, fmt.Errorf("failed to delete DNS router rules, err: %v", err))
}
}
-128
View File
@@ -1,128 +0,0 @@
// Code generated by bpf2go; DO NOT EDIT.
//go:build arm64be || armbe || mips || mips64 || mips64p32 || ppc64 || s390 || s390x || sparc || sparc64
package ebpf
import (
"bytes"
_ "embed"
"fmt"
"io"
"github.com/cilium/ebpf"
)
// loadBpf returns the embedded CollectionSpec for bpf.
func loadBpf() (*ebpf.CollectionSpec, error) {
reader := bytes.NewReader(_BpfBytes)
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
if err != nil {
return nil, fmt.Errorf("can't load bpf: %w", err)
}
return spec, err
}
// loadBpfObjects loads bpf and converts it into a struct.
//
// The following types are suitable as obj argument:
//
// *bpfObjects
// *bpfPrograms
// *bpfMaps
//
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
spec, err := loadBpf()
if err != nil {
return err
}
return spec.LoadAndAssign(obj, opts)
}
// bpfSpecs contains maps and programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type bpfSpecs struct {
bpfProgramSpecs
bpfMapSpecs
}
// bpfSpecs contains programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type bpfProgramSpecs struct {
NbXdpProg *ebpf.ProgramSpec `ebpf:"nb_xdp_prog"`
}
// bpfMapSpecs contains maps before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type bpfMapSpecs struct {
NbFeatures *ebpf.MapSpec `ebpf:"nb_features"`
NbMapDnsIp *ebpf.MapSpec `ebpf:"nb_map_dns_ip"`
NbMapDnsPort *ebpf.MapSpec `ebpf:"nb_map_dns_port"`
NbWgProxySettingsMap *ebpf.MapSpec `ebpf:"nb_wg_proxy_settings_map"`
}
// bpfObjects contains all objects after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfObjects struct {
bpfPrograms
bpfMaps
}
func (o *bpfObjects) Close() error {
return _BpfClose(
&o.bpfPrograms,
&o.bpfMaps,
)
}
// bpfMaps contains all maps after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfMaps struct {
NbFeatures *ebpf.Map `ebpf:"nb_features"`
NbMapDnsIp *ebpf.Map `ebpf:"nb_map_dns_ip"`
NbMapDnsPort *ebpf.Map `ebpf:"nb_map_dns_port"`
NbWgProxySettingsMap *ebpf.Map `ebpf:"nb_wg_proxy_settings_map"`
}
func (m *bpfMaps) Close() error {
return _BpfClose(
m.NbFeatures,
m.NbMapDnsIp,
m.NbMapDnsPort,
m.NbWgProxySettingsMap,
)
}
// bpfPrograms contains all programs after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfPrograms struct {
NbXdpProg *ebpf.Program `ebpf:"nb_xdp_prog"`
}
func (p *bpfPrograms) Close() error {
return _BpfClose(
p.NbXdpProg,
)
}
func _BpfClose(closers ...io.Closer) error {
for _, closer := range closers {
if err := closer.Close(); err != nil {
return err
}
}
return nil
}
// Do not access this directly.
//
//go:embed bpf_bpfeb.o
var _BpfBytes []byte
Binary file not shown.
-128
View File
@@ -1,128 +0,0 @@
// Code generated by bpf2go; DO NOT EDIT.
//go:build 386 || amd64 || amd64p32 || arm || arm64 || loong64 || mips64le || mips64p32le || mipsle || ppc64le || riscv64
package ebpf
import (
"bytes"
_ "embed"
"fmt"
"io"
"github.com/cilium/ebpf"
)
// loadBpf returns the embedded CollectionSpec for bpf.
func loadBpf() (*ebpf.CollectionSpec, error) {
reader := bytes.NewReader(_BpfBytes)
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
if err != nil {
return nil, fmt.Errorf("can't load bpf: %w", err)
}
return spec, err
}
// loadBpfObjects loads bpf and converts it into a struct.
//
// The following types are suitable as obj argument:
//
// *bpfObjects
// *bpfPrograms
// *bpfMaps
//
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
spec, err := loadBpf()
if err != nil {
return err
}
return spec.LoadAndAssign(obj, opts)
}
// bpfSpecs contains maps and programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type bpfSpecs struct {
bpfProgramSpecs
bpfMapSpecs
}
// bpfSpecs contains programs before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type bpfProgramSpecs struct {
NbXdpProg *ebpf.ProgramSpec `ebpf:"nb_xdp_prog"`
}
// bpfMapSpecs contains maps before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type bpfMapSpecs struct {
NbFeatures *ebpf.MapSpec `ebpf:"nb_features"`
NbMapDnsIp *ebpf.MapSpec `ebpf:"nb_map_dns_ip"`
NbMapDnsPort *ebpf.MapSpec `ebpf:"nb_map_dns_port"`
NbWgProxySettingsMap *ebpf.MapSpec `ebpf:"nb_wg_proxy_settings_map"`
}
// bpfObjects contains all objects after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfObjects struct {
bpfPrograms
bpfMaps
}
func (o *bpfObjects) Close() error {
return _BpfClose(
&o.bpfPrograms,
&o.bpfMaps,
)
}
// bpfMaps contains all maps after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfMaps struct {
NbFeatures *ebpf.Map `ebpf:"nb_features"`
NbMapDnsIp *ebpf.Map `ebpf:"nb_map_dns_ip"`
NbMapDnsPort *ebpf.Map `ebpf:"nb_map_dns_port"`
NbWgProxySettingsMap *ebpf.Map `ebpf:"nb_wg_proxy_settings_map"`
}
func (m *bpfMaps) Close() error {
return _BpfClose(
m.NbFeatures,
m.NbMapDnsIp,
m.NbMapDnsPort,
m.NbWgProxySettingsMap,
)
}
// bpfPrograms contains all programs after they have been loaded into the kernel.
//
// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign.
type bpfPrograms struct {
NbXdpProg *ebpf.Program `ebpf:"nb_xdp_prog"`
}
func (p *bpfPrograms) Close() error {
return _BpfClose(
p.NbXdpProg,
)
}
func _BpfClose(closers ...io.Closer) error {
for _, closer := range closers {
if err := closer.Close(); err != nil {
return err
}
}
return nil
}
// Do not access this directly.
//
//go:embed bpf_bpfel.o
var _BpfBytes []byte
Binary file not shown.
@@ -1,52 +0,0 @@
package ebpf
import (
"encoding/binary"
"fmt"
"net/netip"
log "github.com/sirupsen/logrus"
)
const (
mapKeyDNSIP uint32 = 0
mapKeyDNSPort uint32 = 1
)
func (tf *GeneralManager) LoadDNSFwd(ip netip.Addr, dnsPort int) error {
log.Debugf("load eBPF DNS forwarder, watching addr: %s:53, redirect to port: %d", ip, dnsPort)
tf.lock.Lock()
defer tf.lock.Unlock()
err := tf.loadXdp()
if err != nil {
return err
}
if !ip.Is4() {
return fmt.Errorf("eBPF DNS forwarder only supports IPv4, got %s", ip)
}
ip4 := ip.As4()
err = tf.bpfObjs.NbMapDnsIp.Put(mapKeyDNSIP, binary.BigEndian.Uint32(ip4[:]))
if err != nil {
return err
}
err = tf.bpfObjs.NbMapDnsPort.Put(mapKeyDNSPort, uint16(dnsPort))
if err != nil {
return err
}
tf.setFeatureFlag(featureFlagDnsForwarder)
err = tf.bpfObjs.NbFeatures.Put(mapKeyFeatures, tf.featureFlags)
if err != nil {
return err
}
return nil
}
func (tf *GeneralManager) FreeDNSFwd() error {
log.Debugf("free ebpf DNS forwarder")
return tf.unsetFeatureFlag(featureFlagDnsForwarder)
}
-116
View File
@@ -1,116 +0,0 @@
package ebpf
import (
_ "embed"
"net"
"sync"
"github.com/cilium/ebpf/link"
"github.com/cilium/ebpf/rlimit"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/internal/ebpf/manager"
)
const (
mapKeyFeatures uint32 = 0
featureFlagWGProxy = 0b00000001
featureFlagDnsForwarder = 0b00000010
)
var (
singleton manager.Manager
singletonLock = &sync.Mutex{}
)
// required packages libbpf-dev, libc6-dev-i386-amd64-cross
// GeneralManager is used to load multiple eBPF programs with a custom check (if then) done in prog.c
// The manager simply adds a feature (byte) of each program to a map that is shared between the userspace and kernel.
// When packet arrives, the C code checks for each feature (if it is set) and executes each enabled program (e.g., dns_fwd.c and wg_proxy.c).
//
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang-14 bpf src/prog.c -- -I /usr/x86_64-linux-gnu/include
type GeneralManager struct {
lock sync.Mutex
link link.Link
featureFlags uint16
bpfObjs bpfObjects
}
// GetEbpfManagerInstance return a static eBpf Manager instance
func GetEbpfManagerInstance() manager.Manager {
singletonLock.Lock()
defer singletonLock.Unlock()
if singleton != nil {
return singleton
}
singleton = &GeneralManager{}
return singleton
}
func (tf *GeneralManager) setFeatureFlag(feature uint16) {
tf.featureFlags |= feature
}
func (tf *GeneralManager) loadXdp() error {
if tf.link != nil {
return nil
}
// it required for Docker
err := rlimit.RemoveMemlock()
if err != nil {
return err
}
iFace, err := net.InterfaceByName("lo")
if err != nil {
return err
}
// load pre-compiled programs into the kernel.
err = loadBpfObjects(&tf.bpfObjs, nil)
if err != nil {
return err
}
tf.link, err = link.AttachXDP(link.XDPOptions{
Program: tf.bpfObjs.NbXdpProg,
Interface: iFace.Index,
})
if err != nil {
_ = tf.bpfObjs.Close()
tf.link = nil
return err
}
return nil
}
func (tf *GeneralManager) unsetFeatureFlag(feature uint16) error {
tf.lock.Lock()
defer tf.lock.Unlock()
tf.featureFlags &^= feature
if tf.link == nil {
return nil
}
if tf.featureFlags == 0 {
return tf.close()
}
return tf.bpfObjs.NbFeatures.Put(mapKeyFeatures, tf.featureFlags)
}
func (tf *GeneralManager) close() error {
log.Debugf("detach ebpf program ")
err := tf.bpfObjs.Close()
if err != nil {
log.Warnf("failed to close eBpf objects: %s", err)
}
err = tf.link.Close()
tf.link = nil
return err
}
@@ -1,40 +0,0 @@
package ebpf
import (
"testing"
)
func TestManager_setFeatureFlag(t *testing.T) {
mgr := GeneralManager{}
mgr.setFeatureFlag(featureFlagWGProxy)
if mgr.featureFlags != 1 {
t.Errorf("invalid feature state")
}
mgr.setFeatureFlag(featureFlagDnsForwarder)
if mgr.featureFlags != 3 {
t.Errorf("invalid feature state")
}
}
func TestManager_unsetFeatureFlag(t *testing.T) {
mgr := GeneralManager{}
mgr.setFeatureFlag(featureFlagWGProxy)
mgr.setFeatureFlag(featureFlagDnsForwarder)
err := mgr.unsetFeatureFlag(featureFlagWGProxy)
if err != nil {
t.Errorf("unexpected error: %s", err)
}
if mgr.featureFlags != 2 {
t.Errorf("invalid feature state, expected: %d, got: %d", 2, mgr.featureFlags)
}
err = mgr.unsetFeatureFlag(featureFlagDnsForwarder)
if err != nil {
t.Errorf("unexpected error: %s", err)
}
if mgr.featureFlags != 0 {
t.Errorf("invalid feature state, expected: %d, got: %d", 0, mgr.featureFlags)
}
}
-67
View File
@@ -1,67 +0,0 @@
const __u32 map_key_dns_ip = 0;
const __u32 map_key_dns_port = 1;
struct bpf_map_def SEC("maps") nb_map_dns_ip = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(__u32),
.value_size = sizeof(__u32),
.max_entries = 10,
};
struct bpf_map_def SEC("maps") nb_map_dns_port = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(__u32),
.value_size = sizeof(__u16),
.max_entries = 10,
};
__be32 dns_ip = 0;
__be16 dns_port = 0;
// 13568 is 53 in big endian
__be16 GENERAL_DNS_PORT = 13568;
bool read_settings() {
__u16 *port_value;
__u32 *ip_value;
// read dns ip
ip_value = bpf_map_lookup_elem(&nb_map_dns_ip, &map_key_dns_ip);
if(!ip_value) {
return false;
}
dns_ip = htonl(*ip_value);
// read dns port
port_value = bpf_map_lookup_elem(&nb_map_dns_port, &map_key_dns_port);
if (!port_value) {
return false;
}
dns_port = htons(*port_value);
return true;
}
int xdp_dns_fwd(struct iphdr *ip, struct udphdr *udp) {
if (dns_port == 0) {
if(!read_settings()){
return XDP_PASS;
}
// bpf_printk("dns port: %d", ntohs(dns_port));
// bpf_printk("dns ip: %d", ntohl(dns_ip));
}
if (udp->dest == GENERAL_DNS_PORT && ip->daddr == dns_ip) {
udp->dest = dns_port;
// Clear the now-stale checksum; zero means "not computed" for IPv4.
udp->check = 0;
return XDP_PASS;
}
if (udp->source == dns_port && ip->saddr == dns_ip) {
udp->source = GENERAL_DNS_PORT;
udp->check = 0;
return XDP_PASS;
}
return XDP_PASS;
}
-60
View File
@@ -1,60 +0,0 @@
#include <stdbool.h>
#include <linux/if_ether.h> // ETH_P_IP
#include <linux/udp.h>
#include <linux/ip.h>
#include <netinet/in.h>
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include "dns_fwd.c"
#include "wg_proxy.c"
const __u16 flag_feature_wg_proxy = 0b01;
const __u16 flag_feature_dns_fwd = 0b10;
const __u32 map_key_features = 0;
struct bpf_map_def SEC("maps") nb_features = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(__u32),
.value_size = sizeof(__u16),
.max_entries = 10,
};
SEC("xdp")
int nb_xdp_prog(struct xdp_md *ctx) {
__u16 *features;
features = bpf_map_lookup_elem(&nb_features, &map_key_features);
if (!features) {
return XDP_PASS;
}
void *data = (void *)(long)ctx->data;
void *data_end = (void *)(long)ctx->data_end;
struct ethhdr *eth = data;
struct iphdr *ip = (data + sizeof(struct ethhdr));
struct udphdr *udp = (data + sizeof(struct ethhdr) + sizeof(struct iphdr));
// return early if not enough data
if (data + sizeof(struct ethhdr) + sizeof(struct iphdr) + sizeof(struct udphdr) > data_end){
return XDP_PASS;
}
// skip non IPv4 packages
if (eth->h_proto != htons(ETH_P_IP)) {
return XDP_PASS;
}
// skip non UPD packages
if (ip->protocol != IPPROTO_UDP) {
return XDP_PASS;
}
if (*features & flag_feature_dns_fwd) {
xdp_dns_fwd(ip, udp);
}
if (*features & flag_feature_wg_proxy) {
xdp_wg_proxy(ip, udp);
}
return XDP_PASS;
}
char _license[] SEC("license") = "GPL";
-17
View File
@@ -1,17 +0,0 @@
# DNS forwarder
The agent attach the XDP program to the lo device. We can not use fake address in eBPF because the
traffic does not appear in the eBPF program. The program capture the traffic on wg_ip:53 and
overwrite in it the destination port to 5053.
# Debug
The CONFIG_BPF_EVENTS kernel module is required for bpf_printk.
Apply this code to use bpf_printk
```
#define bpf_printk(fmt, ...) \
({ \
char ____fmt[] = fmt; \
bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \
})
```
-60
View File
@@ -1,60 +0,0 @@
const __u32 map_key_proxy_port = 0;
const __u32 map_key_wg_port = 1;
struct bpf_map_def SEC("maps") nb_wg_proxy_settings_map = {
.type = BPF_MAP_TYPE_ARRAY,
.key_size = sizeof(__u32),
.value_size = sizeof(__u16),
.max_entries = 10,
};
__u16 proxy_port = 0;
__u16 wg_port = 0;
bool read_port_settings() {
__u16 *value;
value = bpf_map_lookup_elem(&nb_wg_proxy_settings_map, &map_key_proxy_port);
if (!value) {
return false;
}
proxy_port = *value;
value = bpf_map_lookup_elem(&nb_wg_proxy_settings_map, &map_key_wg_port);
if (!value) {
return false;
}
wg_port = htons(*value);
return true;
}
int xdp_wg_proxy(struct iphdr *ip, struct udphdr *udp) {
if (proxy_port == 0 || wg_port == 0) {
if (!read_port_settings()){
return XDP_PASS;
}
// bpf_printk("proxy port: %d, wg port: %d", proxy_port, wg_port);
}
// 2130706433 = 127.0.0.1
if (ip->daddr != htonl(2130706433)) {
return XDP_PASS;
}
if (udp->source != wg_port){
return XDP_PASS;
}
__be16 new_src_port = udp->dest;
__be16 new_dst_port = htons(proxy_port);
udp->dest = new_dst_port;
udp->source = new_src_port;
// The ports are covered by the UDP checksum. This is an IPv4 loopback hop
// and the payload is already integrity-protected, so clear the checksum (a
// zero UDP checksum means "not computed" for IPv4) rather than leave a
// stale value the kernel would drop as UDP_CSUM.
udp->check = 0;
return XDP_PASS;
}
@@ -1,41 +0,0 @@
package ebpf
import log "github.com/sirupsen/logrus"
const (
mapKeyProxyPort uint32 = 0
mapKeyWgPort uint32 = 1
)
func (tf *GeneralManager) LoadWgProxy(proxyPort, wgPort int) error {
log.Debugf("load ebpf WG proxy")
tf.lock.Lock()
defer tf.lock.Unlock()
err := tf.loadXdp()
if err != nil {
return err
}
err = tf.bpfObjs.NbWgProxySettingsMap.Put(mapKeyProxyPort, uint16(proxyPort))
if err != nil {
return err
}
err = tf.bpfObjs.NbWgProxySettingsMap.Put(mapKeyWgPort, uint16(wgPort))
if err != nil {
return err
}
tf.setFeatureFlag(featureFlagWGProxy)
err = tf.bpfObjs.NbFeatures.Put(mapKeyFeatures, tf.featureFlags)
if err != nil {
return err
}
return nil
}
func (tf *GeneralManager) FreeWGProxy() error {
log.Debugf("free ebpf WG proxy")
return tf.unsetFeatureFlag(featureFlagWGProxy)
}
@@ -1,15 +0,0 @@
//go:build !android
package ebpf
import (
"github.com/netbirdio/netbird/client/internal/ebpf/ebpf"
"github.com/netbirdio/netbird/client/internal/ebpf/manager"
)
// GetEbpfManagerInstance is a wrapper function. This encapsulation is required because if the code import the internal
// ebpf package the Go compiler will include the object files. But it is not supported on Android. It can cause instant
// panic on older Android version.
func GetEbpfManagerInstance() manager.Manager {
return ebpf.GetEbpfManagerInstance()
}
@@ -1,10 +0,0 @@
//go:build !linux || android
package ebpf
import "github.com/netbirdio/netbird/client/internal/ebpf/manager"
// GetEbpfManagerInstance return error because ebpf is not supported on all os
func GetEbpfManagerInstance() manager.Manager {
panic("unsupported os")
}
-11
View File
@@ -1,11 +0,0 @@
package manager
import "net/netip"
// Manager is used to load multiple eBPF programs. E.g., current DNS programs and WireGuard proxy
type Manager interface {
LoadDNSFwd(ip netip.Addr, dnsPort int) error
FreeDNSFwd() error
LoadWgProxy(proxyPort, wgPort int) error
FreeWGProxy() error
}
+11
View File
@@ -6,6 +6,17 @@ import (
"path/filepath"
)
// CheckOnlyOwnerWritable reports an error unless path, and every directory
// leading to it, is owned by an account that can already act with the privileges
// the caller holds, and is writable by nobody else.
//
// Exported for callers outside elevation that read a file while privileged and
// then act on what it says: the same question this package asks of an
// executable, asked of a configuration file.
func CheckOnlyOwnerWritable(path string) error {
return checkOnlyOwnerWritable(path)
}
// trustedSelf returns the path of this executable, provided it is one we are
// willing to have run as root.
//
+190 -52
View File
@@ -14,12 +14,14 @@ import (
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/hashicorp/go-multierror"
"github.com/pion/ice/v4"
"github.com/pion/stun/v3"
log "github.com/sirupsen/logrus"
wgdevice "golang.zx2c4.com/wireguard/device"
"golang.zx2c4.com/wireguard/tun/netstack"
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
@@ -94,6 +96,13 @@ const (
// exec, os.Stat); without this bound a single stuck call freezes handleSync, and
// thus syncMsgMux, for as long as the call hangs (observed multi-minute freezes).
systemInfoTimeout = 15 * time.Second
// dnsForwarderStopTimeout bounds how long stopping the DNS forwarder waits
// for the queries still in flight. One waiting on an unresponsive upstream
// would otherwise hold the stop for the whole upstream timeout, and the
// stop runs with syncMsgMux held. The sockets are closed either way, so
// giving up costs a query that was already failing.
dnsForwarderStopTimeout = 2 * time.Second
)
var ErrResetConnection = fmt.Errorf("reset connection")
@@ -137,6 +146,7 @@ type EngineConfig struct {
RosenpassPermissive bool
ServerSSHAllowed bool
RemoteJobsAllowed bool
EnableSSHRoot *bool
EnableSSHSFTP *bool
EnableSSHLocalPortForwarding *bool
@@ -228,6 +238,12 @@ type Engine struct {
wgInterface WGIface
// wgDevice is a lock-free handle on the WireGuard device behind
// wgInterface. Reaching the device through wgInterface requires
// syncMsgMux, which handleSync holds while it adds and removes peers;
// SetPerformance must stay reachable exactly when that work is stuck.
wgDevice atomic.Pointer[wgdevice.Device]
udpMux *udpmux.UniversalUDPMuxDefault
// networkSerial is the latest CurrentSerial (state ID) of the network sent by the Management service
@@ -257,6 +273,8 @@ type Engine struct {
// checks are the client-applied posture checks that need to be evaluated on the client
checks []*mgmProto.Checks
infoSource system.InfoSource
relayManager *relayClient.Manager
stateManager *statemanager.Manager
portForwardManager *portforward.Manager
@@ -320,6 +338,10 @@ type localIpUpdater interface {
UpdateLocalIPs() error
}
// overlayRebind rebuilds one subsystem's sockets on the current interface. The
// error it returns names its own subsystem, since the caller can only log it.
type overlayRebind func() error
// NewEngine creates a new Connection Engine with probes attached
func NewEngine(
clientCtx context.Context,
@@ -637,10 +659,7 @@ func (e *Engine) Start(netbirdConfig *mgmProto.NetbirdConfig, mgmtURL *url.URL)
log.Errorf("failed to pull up wgInterface [%s]: %s", e.wgInterface.Name(), err.Error())
return fmt.Errorf("up wg interface: %w", err)
}
// Set up notrack rules immediately after proxy is listening to prevent
// conntrack entries from being created before the rules are in place
e.setupWGProxyNoTrack()
e.wgDevice.Store(e.wgInterface.GetWGDevice())
// Start after interface is up since port may have been resolved from 0 or changed if occupied
e.shutdownWg.Add(1)
@@ -744,6 +763,11 @@ func (e *Engine) initFirewall() error {
return fmt.Errorf("set firewall: %w", err)
}
// TODO: the firewall backends dedup filter rules by content, so a
// management route ACL with identical content would collapse onto the
// untracked drop rules installed here, and a later management delete
// could remove them. Needs backend refcounting or per-consumer key
// namespacing.
if e.config.BlockLANAccess {
e.blockLanAccess()
}
@@ -756,14 +780,14 @@ func (e *Engine) initFirewall() error {
port := firewallManager.Port{Values: []uint16{uint16(rosenpassPort)}}
// IPv4-only: rosenpass peers connect via AllowedIps[0] which is always v4.
if _, err := e.firewall.AddPeerFiltering(
if _, err := e.firewall.AddFilterRule(
nil,
net.IP{0, 0, 0, 0},
[]netip.Prefix{netip.PrefixFrom(netip.IPv4Unspecified(), 0)},
firewallManager.Network{},
firewallManager.ProtocolUDP,
nil,
&port,
firewallManager.ActionAccept,
"",
); err != nil {
log.Errorf("failed to allow rosenpass interface traffic: %v", err)
return nil
@@ -774,23 +798,6 @@ func (e *Engine) initFirewall() error {
return nil
}
// setupWGProxyNoTrack configures connection tracking exclusion for WireGuard proxy traffic.
// This prevents conntrack/MASQUERADE from affecting loopback traffic between WireGuard and the eBPF proxy.
func (e *Engine) setupWGProxyNoTrack() {
if e.firewall == nil {
return
}
proxyPort := e.wgInterface.GetProxyPort()
if proxyPort == 0 {
return
}
if err := e.firewall.SetupEBPFProxyNoTrack(proxyPort, uint16(e.config.WgPort)); err != nil {
log.Warnf("failed to setup ebpf proxy notrack: %v", err)
}
}
func (e *Engine) blockLanAccess() {
if e.config.BlockInbound {
// no need to set up extra deny rules if inbound is already blocked in general
@@ -813,7 +820,7 @@ func (e *Engine) blockLanAccess() {
if network.Addr().Is6() {
source = v6
}
if _, err := e.firewall.AddRouteFiltering(
if _, err := e.firewall.AddFilterRule(
nil,
[]netip.Prefix{source},
firewallManager.Network{Prefix: network},
@@ -1033,7 +1040,11 @@ func (e *Engine) handleSync(update *mgmProto.SyncResponse) error {
// back to empty if the FQDN doesn't have the expected shape.
dnsName = extractDNSDomainFromFQDN(pc.GetFqdn())
}
result, err := nbnetworkmap.EnvelopeToNetworkMap(e.ctx, envelope, localKey, dnsName)
// With the firewall disabled there is no ACL manager to program, so
// RoutesFirewallRules would be built and then dropped. On a peer that
// routes many network resources that is the single most expensive
// step of the sync.
result, err := nbnetworkmap.EnvelopeToNetworkMap(e.ctx, envelope, localKey, dnsName, e.config.DisableFirewall)
if err != nil {
return fmt.Errorf("decode network map envelope: %w", err)
}
@@ -1224,9 +1235,7 @@ func (e *Engine) updateChecksIfNew(checks []*mgmProto.Checks) error {
if isChecksEqual(e.checks, checks) {
return nil
}
e.checks = checks
info, ok := system.GetInfoWithChecksTimeout(e.ctx, systemInfoTimeout, checks, e.overlayAddresses()...)
info, ok := e.infoSource.Refresh(e.ctx, systemInfoTimeout, checks, e.overlayAddresses()...)
if !ok {
// Gathering timed out; skip the meta sync this cycle rather than blocking the
// sync loop (and syncMsgMux) on a stuck system call. A later sync will retry.
@@ -1237,6 +1246,7 @@ func (e *Engine) updateChecksIfNew(checks []*mgmProto.Checks) error {
if err := e.mgmClient.SyncMeta(info); err != nil {
return fmt.Errorf("could not sync meta: error %s", err)
}
e.checks = checks
return nil
}
@@ -1259,9 +1269,32 @@ func (e *Engine) applyInfoFlags(info *system.Info) {
e.config.EnableSSHLocalPortForwarding,
e.config.EnableSSHRemotePortForwarding,
e.config.DisableSSHAuth,
&e.config.RemoteJobsAllowed,
)
}
func (e *Engine) currentSystemInfo(ctx context.Context) *system.Info {
info := e.infoSource.Current(ctx, e.overlayAddresses()...)
e.applyInfoFlags(info)
return info
}
// syncInfoFunc returns the info callback for the management sync stream. The
// first connect sends the info refreshed right before it instead of gathering
// again; every reconnect gathers a fresh one. The stream retry loop calls the
// callback sequentially, so the handoff needs no synchronization.
func (e *Engine) syncInfoFunc(refreshed *system.Info) func(ctx context.Context) *system.Info {
return func(ctx context.Context) *system.Info {
if refreshed == nil {
return e.currentSystemInfo(ctx)
}
info := refreshed
refreshed = nil
e.applyInfoFlags(info)
return info
}
}
// overlayAddresses returns our own WireGuard overlay address (v4 and v6) so it
// can be excluded from the reported network addresses; the interface coming and
// going otherwise churns the peer meta on the management server.
@@ -1344,6 +1377,13 @@ func (e *Engine) receiveJobEvents() {
ID: msg.ID,
Status: mgmProto.JobStatus_failed,
}
// Remote jobs are an explicit opt-in. When not enabled on this
// peer, every job is refused before any work is done.
if !e.config.RemoteJobsAllowed {
log.Warnf("refusing remote job: remote jobs are not enabled on this peer (enable with --allow-remote-jobs)")
resp.Reason = []byte("remote jobs are not enabled on this peer")
return &resp
}
switch params := msg.WorkloadParameters.(type) {
case *mgmProto.JobRequest_Bundle:
bundleResult, err := e.handleBundle(params.Bundle)
@@ -1373,7 +1413,25 @@ func (e *Engine) receiveJobEvents() {
}
func (e *Engine) handleBundle(params *mgmProto.BundleParameters) (*mgmProto.JobResponse_Bundle, error) {
log.Infof("handle remote debug bundle request: %s", params.String())
// The upload URL can carry a host, credentials, or query tokens, so it is
// kept out of the info-level line; the full parameters stay available at
// debug level for troubleshooting.
log.Infof("handle remote debug bundle request: anonymize=%v anonymize_level=%q log_file_count=%d bundle_for=%v bundle_for_time=%d",
params.GetAnonymize(), params.GetAnonymizeLevel(), params.GetLogFileCount(), params.GetBundleFor(), params.GetBundleForTime())
log.Debugf("remote debug bundle request parameters: %s", params.String())
// Resolve the upload destination: an MDM override, when set, takes
// precedence over the management-supplied URL. Both are validated the same
// way; an empty result falls back to the default upload server downstream.
uploadURL := params.GetUploadUrl()
if override := e.config.ProfileConfig.DebugBundleUploadURL; override != "" {
log.Infof("using MDM debug bundle upload URL override instead of the management-supplied value")
uploadURL = override
}
if err := validateBundleUploadURL(uploadURL); err != nil {
return nil, err
}
syncResponse, err := e.GetLatestSyncResponse()
if err != nil {
log.Warnf("get latest sync response: %v", err)
@@ -1401,7 +1459,7 @@ func (e *Engine) handleBundle(params *mgmProto.BundleParameters) (*mgmProto.JobR
waitFor := time.Duration(params.BundleForTime) * time.Minute
uploadKey, err := e.jobExecutor.BundleJob(e.ctx, bundleDeps, bundleJobParams, waitFor, e.config.ProfileConfig.ManagementURL.String())
uploadKey, err := e.jobExecutor.BundleJob(e.ctx, bundleDeps, bundleJobParams, waitFor, e.config.ProfileConfig.ManagementURL.String(), uploadURL)
if err != nil {
return nil, err
}
@@ -1414,21 +1472,27 @@ func (e *Engine) handleBundle(params *mgmProto.BundleParameters) (*mgmProto.JobR
return response, nil
}
// validateBundleUploadURL sanity-checks a management-supplied upload URL for a
// remote debug bundle job. It delegates to profilemanager.ValidateBundleUploadURL
// so the executor and the MDM policy override share one definition of the rule
// (empty accepted; otherwise a well-formed https URL with a host) and cannot
// drift. The host is deliberately left unconstrained pending a decision on
// management-directed uploads.
func validateBundleUploadURL(raw string) error {
return profilemanager.ValidateBundleUploadURL(raw)
}
// receiveManagementEvents connects to the Management Service event stream to receive updates from the management service
// E.g. when a new peer has been registered and we are allowed to connect to it.
func (e *Engine) receiveManagementEvents() {
e.shutdownWg.Add(1)
go func() {
defer e.shutdownWg.Done()
info, ok := system.GetInfoWithChecksTimeout(e.ctx, systemInfoTimeout, e.checks, e.overlayAddresses()...)
info, ok := e.infoSource.Refresh(e.ctx, systemInfoTimeout, e.checks, e.overlayAddresses()...)
if !ok {
// Gathering timed out; connect the stream with base info so management
// connectivity still comes up rather than blocking here.
info = system.GetInfo(e.ctx)
log.Warnf("posture checks not refreshed before the sync connect, sending the previous results")
}
e.applyInfoFlags(info)
err := e.mgmClient.Sync(e.ctx, info, e.handleSync)
err := e.mgmClient.Sync(e.ctx, e.syncInfoFunc(info), e.handleSync)
if err != nil {
// happens if management is unavailable for a long time.
// We want to cancel the operation of the whole client
@@ -2072,6 +2136,10 @@ func (e *Engine) close() {
log.Debugf("removing Netbird interface %s", e.config.WgIfaceName)
if e.wgInterface != nil {
// Drop the handle before the close starts: a retune that loads it
// afterwards would touch a device on its way out and report success
// for an engine that is already gone.
e.wgDevice.Store(nil)
if err := e.wgInterface.Close(); err != nil {
log.Errorf("failed closing Netbird interface %s %v", e.config.WgIfaceName, err)
}
@@ -2110,10 +2178,7 @@ func (e *Engine) close() {
}
func (e *Engine) newWgIface() (*iface.WGIface, error) {
transportNet, err := e.newStdNet()
if err != nil {
log.Errorf("failed to create pion's stdnet: %s", err)
}
transportNet := e.newStdNet()
opts := iface.WGIFaceOpts{
IFaceName: e.config.WgIfaceName,
@@ -2231,15 +2296,16 @@ type Performance struct {
}
// SetPerformance applies the given tuning to this engine's live Device.
//
// It deliberately does not take syncMsgMux. Raising the buffer pool cap is the
// recovery path for a device whose pool is exhausted, and an exhausted pool
// blocks peer removal inside handleSync, which holds syncMsgMux for as long as
// it stays blocked. Taking the lock here would make the retune unreachable in
// the one situation that needs it.
func (e *Engine) SetPerformance(t Performance) error {
e.syncMsgMux.Lock()
defer e.syncMsgMux.Unlock()
if e.wgInterface == nil {
return fmt.Errorf("wg interface not initialized")
}
dev := e.wgInterface.GetWGDevice()
dev := e.wgDevice.Load()
if dev == nil {
return fmt.Errorf("wg device not initialized")
return errors.New("wg device not initialized")
}
if t.PreallocatedBuffersPerPool != nil {
dev.SetPreallocatedBuffersPerPool(*t.PreallocatedBuffersPerPool)
@@ -2460,7 +2526,72 @@ func (e *Engine) RenewTun(fd int) error {
return fmt.Errorf("wireguard interface not initialized")
}
return wgInterface.RenewTun(fd)
if err := wgInterface.RenewTun(fd); err != nil {
return err
}
e.rebindOverlayListeners()
return nil
}
// rebindOverlayListeners gives the servers that listen on an overlay address
// sockets on the interface as it is now.
//
// A socket belongs to the interface generation it was created on. Renewing the
// TUN builds a new interface and moves the overlay addresses to it, which
// leaves the old sockets in LISTEN with the uspfilter still logging packets
// arriving for them, while every accept fails with EINVAL for the life of the
// socket: from the outside the server looks alive and answers nothing. On
// Android this happens during a normal startup, where the first TUN is
// established before the routes are known and replaced once they arrive.
//
// Rebinding costs whatever those sockets were carrying, which the renewal has
// already broken. Errors are logged rather than returned: the renewal itself
// succeeded, and failing it would hand the caller a working interface and an
// error.
func (e *Engine) rebindOverlayListeners() {
e.syncMsgMux.Lock()
defer e.syncMsgMux.Unlock()
for _, rebind := range e.overlayRebinds() {
if err := rebind(); err != nil {
log.Errorf("after TUN renewal: %v", err)
}
}
}
// overlayRebinds is every subsystem of this engine that holds sockets bound to
// an overlay address, and how to rebuild each one's.
//
// A subsystem that starts listening on an overlay address belongs in this list.
// Leaving it out costs nothing that review would notice and produces a listener
// that stays in LISTEN, is logged as receiving packets, and refuses every
// connection for the life of the process.
func (e *Engine) overlayRebinds() []overlayRebind {
return []overlayRebind{
e.restartSSHListeners,
e.restartDNSForwarder,
}
}
// restartDNSForwarder rebuilds the DNS forwarder serving the same domains.
// No-op when it is not running. See Engine.rebindOverlayListeners.
func (e *Engine) restartDNSForwarder() error {
if e.dnsForwardMgr == nil {
return nil
}
// Read from the forwarder before it goes away, so the replacement serves
// the domains in force now rather than a copy kept somewhere else.
entries := e.dnsForwardMgr.Domains()
e.stopDNSForwarder()
// Both halves log their own failures, so the only thing left to report is
// the outcome: a start that failed left the manager nil, and the forwarder
// is now down rather than merely rebound.
e.startDNSForwarder(entries)
if e.dnsForwardMgr == nil {
return errors.New("rebind DNS forwarder: it did not come back up")
}
return nil
}
// updateDNSForwarder start or stop the DNS forwarder based on the domains and the feature flag
@@ -2506,7 +2637,14 @@ func (e *Engine) stopDNSForwarder() {
return
}
if err := e.dnsForwardMgr.Stop(context.Background()); err != nil {
// Bounded because the shutdown waits for queries still in flight, and one
// waiting on an unresponsive upstream holds it for as long as that lookup
// is allowed to take. This runs with syncMsgMux held, so that wait is one
// the whole engine spends.
ctx, cancel := context.WithTimeout(context.Background(), dnsForwarderStopTimeout)
defer cancel()
if err := e.dnsForwardMgr.Stop(ctx); err != nil {
log.Errorf("failed to stop DNS forward: %v", err)
}
@@ -2621,7 +2759,7 @@ func (e *Engine) updateForwardRules(rules []*mgmProto.ForwardingRule) ([]firewal
var merr *multierror.Error
forwardingRules := make([]firewallManager.ForwardRule, 0, len(rules))
for _, rule := range rules {
proto, err := convertToFirewallProtocol(rule.GetProtocol())
proto, err := acl.ConvertToFirewallProtocol(rule.GetProtocol())
if err != nil {
merr = multierror.Append(merr, fmt.Errorf("failed to convert protocol '%s': %w", rule.GetProtocol(), err))
continue
+36
View File
@@ -0,0 +1,36 @@
package internal
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestValidateBundleUploadURL covers the sanity check applied to a
// management-supplied upload URL before a remote debug bundle is generated.
func TestValidateBundleUploadURL(t *testing.T) {
for _, tc := range []struct {
name string
raw string
wantErr bool
}{
{name: "empty falls back to default", raw: ""},
{name: "https with host", raw: "https://upload.debug.netbird.io/upload"},
{name: "https self-hosted host", raw: "https://upload.example.com"},
{name: "plaintext rejected", raw: "http://upload.example.com", wantErr: true},
{name: "missing host rejected", raw: "https:///upload", wantErr: true},
{name: "port-only authority rejected", raw: "https://:443", wantErr: true},
{name: "non-url scheme rejected", raw: "ftp://upload.example.com", wantErr: true},
{name: "garbage rejected", raw: "://not a url", wantErr: true},
} {
t.Run(tc.name, func(t *testing.T) {
err := validateBundleUploadURL(tc.raw)
if tc.wantErr {
require.Error(t, err, "an invalid upload URL must be rejected")
return
}
assert.NoError(t, err, "a valid or empty upload URL must be accepted")
})
}
}
+16 -12
View File
@@ -12,12 +12,12 @@ import (
"testing"
"time"
"go.uber.org/mock/gomock"
"github.com/google/uuid"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel"
"go.uber.org/mock/gomock"
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
"google.golang.org/grpc"
"google.golang.org/grpc/keepalive"
@@ -27,6 +27,7 @@ import (
"github.com/netbirdio/netbird/client/iface/wgaddr"
"github.com/netbirdio/netbird/client/internal/dns"
"github.com/netbirdio/netbird/client/internal/peer"
"github.com/netbirdio/netbird/client/internal/profilemanager"
nbssh "github.com/netbirdio/netbird/client/ssh"
"github.com/netbirdio/netbird/client/system"
nbdns "github.com/netbirdio/netbird/dns"
@@ -81,6 +82,7 @@ func TestEngine_SSH(t *testing.T) {
WgAddr: wgaddr.MustParseWGAddress("100.64.0.1/24"),
WgPrivateKey: key,
WgPort: 33100,
IFaceBlackList: profilemanager.DefaultInterfaceBlacklist,
ServerSSHAllowed: true,
MTU: iface.DefaultMTU,
SSHKey: sshKey,
@@ -193,7 +195,7 @@ func TestEngine_Sync(t *testing.T) {
// feed updates to Engine via mocked Management client
updates := make(chan *mgmtProto.SyncResponse)
defer close(updates)
syncFunc := func(ctx context.Context, info *system.Info, msgHandler func(msg *mgmtProto.SyncResponse) error) error {
syncFunc := func(ctx context.Context, _ func(context.Context) *system.Info, msgHandler func(msg *mgmtProto.SyncResponse) error) error {
for msg := range updates {
err := msgHandler(msg)
if err != nil {
@@ -204,11 +206,12 @@ func TestEngine_Sync(t *testing.T) {
}
relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String(), iface.DefaultMTU)
engine := NewEngine(ctx, cancel, &EngineConfig{
WgIfaceName: "utun103",
WgAddr: wgaddr.MustParseWGAddress("100.64.0.1/24"),
WgPrivateKey: key,
WgPort: 33100,
MTU: iface.DefaultMTU,
WgIfaceName: "utun103",
WgAddr: wgaddr.MustParseWGAddress("100.64.0.1/24"),
WgPrivateKey: key,
WgPort: 33100,
IFaceBlackList: profilemanager.DefaultInterfaceBlacklist,
MTU: iface.DefaultMTU,
}, EngineServices{
SignalClient: &signal.MockClient{},
MgmClient: &mgmt.MockClient{SyncFunc: syncFunc},
@@ -412,11 +415,12 @@ func createEngine(ctx context.Context, cancel context.CancelFunc, setupKey strin
wgPort := 33100 + i
conf := &EngineConfig{
WgIfaceName: ifaceName,
WgAddr: wgaddr.MustParseWGAddress(resp.PeerConfig.Address),
WgPrivateKey: key,
WgPort: wgPort,
MTU: iface.DefaultMTU,
WgIfaceName: ifaceName,
WgAddr: wgaddr.MustParseWGAddress(resp.PeerConfig.Address),
WgPrivateKey: key,
WgPort: wgPort,
IFaceBlackList: profilemanager.DefaultInterfaceBlacklist,
MTU: iface.DefaultMTU,
}
relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String(), iface.DefaultMTU)
+32 -4
View File
@@ -24,6 +24,8 @@ type sshServer interface {
Stop() error
GetStatus() (bool, []sshserver.SessionInfo)
UpdateSSHAuth(config *sshauth.Config)
JWTConfig() *sshserver.JWTConfig
AuthConfig() *sshauth.Config
}
func (e *Engine) setupSSHPortRedirection() error {
@@ -77,7 +79,7 @@ func (e *Engine) updateSSH(sshConf *mgmProto.SSHConfig) error {
if e.config.DisableSSHAuth != nil && *e.config.DisableSSHAuth {
log.Info("starting SSH server without JWT authentication (authentication disabled by config)")
return e.startSSHServer(nil)
return e.startSSHServer(nil, nil)
}
if protoJWT := sshConf.GetJwtConfig(); protoJWT != nil {
@@ -95,7 +97,7 @@ func (e *Engine) updateSSH(sshConf *mgmProto.SSHConfig) error {
MaxTokenAge: protoJWT.GetMaxTokenAge(),
}
return e.startSSHServer(jwtConfig)
return e.startSSHServer(jwtConfig, nil)
}
return errors.New("SSH server requires valid JWT configuration")
@@ -231,8 +233,33 @@ func (e *Engine) cleanupSSHConfig() {
}
}
// startSSHServer initializes and starts the SSH server with proper configuration.
func (e *Engine) startSSHServer(jwtConfig *sshserver.JWTConfig) error {
// restartSSHListeners rebuilds the SSH server so it listens on new sockets, on
// the same terms it was started with. No-op when it is not running. See
// Engine.rebindOverlayListeners for why this is needed.
func (e *Engine) restartSSHListeners() error {
if e.sshServer == nil {
return nil
}
// Read from the server before it goes away. A rebuilt one starts with an
// empty authorizer, which fails closed, so without carrying the
// authorization over every JWT login is refused until the next network map
// happens to bring one.
jwtConfig, authConfig := e.sshServer.JWTConfig(), e.sshServer.AuthConfig()
if err := e.stopSSHServer(); err != nil {
return fmt.Errorf("rebind SSH listeners: %w", err)
}
if err := e.startSSHServer(jwtConfig, authConfig); err != nil {
return fmt.Errorf("rebind SSH listeners: %w", err)
}
return nil
}
// startSSHServer initializes and starts the SSH server with proper
// configuration. authConfig is the fine-grained authorization to open with, and
// is applied before the server accepts anything: a server that starts listening
// with an empty authorizer refuses the logins that arrive in the meantime.
// Nil leaves it as management has not sent one yet.
func (e *Engine) startSSHServer(jwtConfig *sshserver.JWTConfig, authConfig *sshauth.Config) error {
if e.wgInterface == nil {
return errors.New("wg interface not initialized")
}
@@ -240,6 +267,7 @@ func (e *Engine) startSSHServer(jwtConfig *sshserver.JWTConfig) error {
serverConfig := &sshserver.Config{
HostKeyPEM: e.config.SSHKey,
JWT: jwtConfig,
Auth: authConfig,
}
server := sshserver.New(serverConfig)
+1 -1
View File
@@ -6,6 +6,6 @@ import (
"github.com/netbirdio/netbird/client/internal/stdnet"
)
func (e *Engine) newStdNet() (*stdnet.Net, error) {
func (e *Engine) newStdNet() *stdnet.Net {
return stdnet.NewNet(e.clientCtx, e.config.IFaceBlackList)
}
+1 -1
View File
@@ -2,6 +2,6 @@ package internal
import "github.com/netbirdio/netbird/client/internal/stdnet"
func (e *Engine) newStdNet() (*stdnet.Net, error) {
func (e *Engine) newStdNet() *stdnet.Net {
return stdnet.NewNetWithDiscover(e.clientCtx, e.mobileDep.IFaceDiscover, e.config.IFaceBlackList)
}
+134 -16
View File
@@ -2,6 +2,7 @@ package internal
import (
"context"
"errors"
"fmt"
"net"
"net/netip"
@@ -31,6 +32,7 @@ import (
icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
"github.com/netbirdio/netbird/client/internal/profilemanager"
"github.com/netbirdio/netbird/client/internal/routemanager"
"github.com/netbirdio/netbird/client/system"
nbdns "github.com/netbirdio/netbird/dns"
"github.com/netbirdio/netbird/monotime"
"github.com/netbirdio/netbird/route"
@@ -63,7 +65,6 @@ type MockWGIface struct {
GetStatsFunc func() (map[string]configurer.WGStats, error)
GetInterfaceGUIDStringFunc func() (string, error)
GetProxyFunc func() wgproxy.Proxy
GetProxyPortFunc func() uint16
GetNetFunc func() *netstack.Net
LastActivitiesFunc func() map[string]monotime.Time
}
@@ -160,13 +161,6 @@ func (m *MockWGIface) GetProxy() wgproxy.Proxy {
return m.GetProxyFunc()
}
func (m *MockWGIface) GetProxyPort() uint16 {
if m.GetProxyPortFunc != nil {
return m.GetProxyPortFunc()
}
return 0
}
func (m *MockWGIface) GetNet() *netstack.Net {
return m.GetNetFunc()
}
@@ -253,6 +247,118 @@ func TestEngine_SSHServerConsistency(t *testing.T) {
})
}
func TestEngine_FirstSyncInfoCarriesLoginChecks(t *testing.T) {
key, err := wgtypes.GeneratePrivateKey()
require.NoError(t, err)
exe, err := os.Executable()
require.NoError(t, err)
ctx, cancel := context.WithCancel(CtxInitState(context.Background()))
defer cancel()
infos := make(chan *system.Info, 1)
mgmClient := &mgmt.MockClient{
SyncFunc: func(ctx context.Context, getInfo func(context.Context) *system.Info, _ func(*mgmtProto.SyncResponse) error) error {
infos <- getInfo(ctx)
return nil
},
}
relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String(), iface.DefaultMTU)
engine := NewEngine(ctx, cancel, &EngineConfig{
WgIfaceName: "utun104",
WgAddr: wgaddr.MustParseWGAddress("100.64.0.1/24"),
WgPrivateKey: key,
WgPort: 33100,
MTU: iface.DefaultMTU,
}, EngineServices{
SignalClient: &signal.MockClient{},
MgmClient: mgmClient,
RelayManager: relayMgr,
StatusRecorder: peer.NewRecorder("https://mgm"),
Checks: []*mgmtProto.Checks{{Files: []string{exe}}},
}, MobileDependency{})
engine.receiveManagementEvents()
select {
case info := <-infos:
require.Len(t, info.Files, 1)
assert.Equal(t, exe, info.Files[0].Path)
assert.True(t, info.Files[0].Exist)
case <-time.After(20 * time.Second):
t.Fatal("timeout waiting for the first sync info")
}
engine.shutdownWg.Wait()
}
func TestEngine_SyncInfoFuncReusesRefreshedInfoOnce(t *testing.T) {
engine := &Engine{config: &EngineConfig{}}
refreshed := &system.Info{Hostname: "from-refresh"}
getInfo := engine.syncInfoFunc(refreshed)
first := getInfo(context.Background())
assert.Same(t, refreshed, first, "the first connect should send the refreshed info instead of gathering again")
second := getInfo(context.Background())
assert.NotSame(t, refreshed, second, "the reconnect should gather a fresh info")
assert.NotEqual(t, "from-refresh", second.Hostname, "the fresh info should not carry the refreshed hostname")
}
func TestEngine_SyncInfoFuncGathersWhenRefreshFailed(t *testing.T) {
engine := &Engine{config: &EngineConfig{}}
info := engine.syncInfoFunc(nil)(context.Background())
require.NotNil(t, info, "a failed refresh should fall back to gathering the info")
assert.NotEmpty(t, info.Hostname, "the gathered info should carry the hostname")
}
func TestEngine_UpdateChecksIfNewRetriesAfterFailedSyncMeta(t *testing.T) {
key, err := wgtypes.GeneratePrivateKey()
require.NoError(t, err)
exe, err := os.Executable()
require.NoError(t, err)
ctx, cancel := context.WithCancel(CtxInitState(context.Background()))
defer cancel()
syncMetaCalls := 0
mgmClient := &mgmt.MockClient{
SyncMetaFunc: func(*system.Info) error {
syncMetaCalls++
if syncMetaCalls == 1 {
return errors.New("management unavailable")
}
return nil
},
}
relayMgr := relayClient.NewManager(ctx, nil, key.PublicKey().String(), iface.DefaultMTU)
engine := NewEngine(ctx, cancel, &EngineConfig{
WgIfaceName: "utun105",
WgAddr: wgaddr.MustParseWGAddress("100.64.0.1/24"),
WgPrivateKey: key,
WgPort: 33100,
MTU: iface.DefaultMTU,
}, EngineServices{
SignalClient: &signal.MockClient{},
MgmClient: mgmClient,
RelayManager: relayMgr,
StatusRecorder: peer.NewRecorder("https://mgm"),
}, MobileDependency{})
checks := []*mgmtProto.Checks{{Files: []string{exe}}}
require.Error(t, engine.updateChecksIfNew(checks))
require.NoError(t, engine.updateChecksIfNew(checks))
require.NoError(t, engine.updateChecksIfNew(checks))
assert.Equal(t, 2, syncMetaCalls)
}
func TestEngine_UpdateNetworkMap(t *testing.T) {
// test setup
key, err := wgtypes.GeneratePrivateKey()
@@ -582,10 +688,7 @@ func TestEngine_UpdateNetworkMapWithRoutes(t *testing.T) {
StatusRecorder: peer.NewRecorder("https://mgm"),
}, MobileDependency{})
engine.ctx = ctx
newNet, err := stdnet.NewNet(context.Background(), nil)
if err != nil {
t.Fatal(err)
}
newNet := stdnet.NewNet(context.Background(), profilemanager.DefaultInterfaceBlacklist)
opts := iface.WGIFaceOpts{
IFaceName: wgIfaceName,
@@ -790,10 +893,7 @@ func TestEngine_UpdateNetworkMapWithDNSUpdate(t *testing.T) {
}, MobileDependency{})
engine.ctx = ctx
newNet, err := stdnet.NewNet(context.Background(), nil)
if err != nil {
t.Fatal(err)
}
newNet := stdnet.NewNet(context.Background(), profilemanager.DefaultInterfaceBlacklist)
opts := iface.WGIFaceOpts{
IFaceName: wgIfaceName,
Address: wgaddr.MustParseWGAddress(wgAddr),
@@ -1393,3 +1493,21 @@ func TestOverlayAddrsFromAllowedIPs(t *testing.T) {
})
}
}
func TestEngine_SyncResponsePersistence(t *testing.T) {
e := &Engine{}
_, err := e.GetLatestSyncResponse()
require.Error(t, err, "persistence is disabled by default")
e.SetSyncResponsePersistence(true)
e.persistSyncResponse(&mgmtProto.SyncResponse{NetworkMap: &mgmtProto.NetworkMap{Serial: 7}})
got, err := e.GetLatestSyncResponse()
require.NoError(t, err)
assert.Equal(t, uint64(7), got.GetNetworkMap().GetSerial())
e.SetSyncResponsePersistence(false)
_, err = e.GetLatestSyncResponse()
require.Error(t, err)
}
-1
View File
@@ -28,7 +28,6 @@ type wgIfaceBase interface {
Up() (*udpmux.UniversalUDPMuxDefault, error)
UpdateAddr(newAddr wgaddr.Address) error
GetProxy() wgproxy.Proxy
GetProxyPort() uint16
UpdatePeer(peerKey string, allowedIps []netip.Prefix, keepAlive time.Duration, endpoint *net.UDPAddr, preSharedKey *wgtypes.Key) error
RemoveEndpointAddress(key string) error
RemovePeer(peerKey string) error
+8 -4
View File
@@ -24,14 +24,14 @@ type RulePair struct {
type Manager struct {
dnatFirewall DNATFirewall
rules map[string]RulePair // keys is the ID of the ForwardRule
rules map[firewall.RuleID]RulePair
rulesMu sync.Mutex
}
func NewManager(dnatFirewall DNATFirewall) *Manager {
return &Manager{
dnatFirewall: dnatFirewall,
rules: make(map[string]RulePair),
rules: make(map[firewall.RuleID]RulePair),
}
}
@@ -41,7 +41,7 @@ func (h *Manager) Update(forwardRules []firewall.ForwardRule) error {
var mErr *multierror.Error
toDelete := make(map[string]RulePair, len(h.rules))
toDelete := make(map[firewall.RuleID]RulePair, len(h.rules))
for id, r := range h.rules {
toDelete[id] = r
}
@@ -59,6 +59,10 @@ func (h *Manager) Update(forwardRules []firewall.ForwardRule) error {
mErr = multierror.Append(mErr, fmt.Errorf("add forward rule '%s': %v", fwdRule.String(), err))
continue
}
if rule == nil {
mErr = multierror.Append(mErr, fmt.Errorf("add forward rule '%s': backend returned no rule", fwdRule.String()))
continue
}
log.Infof("forward rule has been added '%s'", fwdRule)
h.rules[id] = RulePair{
ForwardRule: fwdRule,
@@ -90,7 +94,7 @@ func (h *Manager) Close() error {
}
}
h.rules = make(map[string]RulePair)
h.rules = make(map[firewall.RuleID]RulePair)
return nberrors.FormatErrorOrNil(mErr)
}
+3 -3
View File
@@ -14,11 +14,11 @@ var (
)
type MocFwRule struct {
id string
id firewall.RuleID
}
func (m *MocFwRule) ID() string {
return string(m.id)
func (m *MocFwRule) ID() firewall.RuleID {
return m.id
}
type MockDNATFirewall struct {
+1 -1
View File
@@ -38,7 +38,7 @@ func asDaemon(t *testing.T, id Identity) {
prevID, prevKnown, prevDelegate := selfIdentity, selfKnown, selfMayDelegate
t.Cleanup(func() { selfIdentity, selfKnown, selfMayDelegate = prevID, prevKnown, prevDelegate })
selfIdentity, selfKnown = id, true
selfMayDelegate = !id.IsPrivileged()
selfMayDelegate = mayDelegate(id)
}
func TestCallerIdentity_DirectConnections(t *testing.T) {
+19 -6
View File
@@ -18,7 +18,8 @@ import (
"google.golang.org/grpc/peer"
)
// Well-known Windows SIDs that identify a fully privileged principal.
// Well-known Windows SIDs. Only LocalSystem and BUILTIN\Administrators identify a
// privileged principal; the service accounts are shared by unrelated services.
const (
sidLocalSystem = "S-1-5-18" // NT AUTHORITY\SYSTEM
sidLocalService = "S-1-5-19" // NT AUTHORITY\LOCAL SERVICE
@@ -67,9 +68,9 @@ func (i Identity) IsWindows() bool {
// user-to-root boundary.
//
// On Windows the decision comes from the caller's token rather than from
// account names or group RIDs: an elevated token, one of the service accounts
// the daemon itself may run as, or a token with BUILTIN\Administrators
// enabled. A UAC-filtered administrator has that group marked deny-only, and
// account names or group RIDs: an elevated token, the LocalSystem SID, or a
// token with BUILTIN\Administrators enabled. LocalService and NetworkService
// are not privileged by SID. A UAC-filtered administrator has that group marked deny-only, and
// deny-only groups are dropped when the identity is captured, so such a
// caller is correctly reported as unprivileged. Domain group memberships
// (Domain Admins and friends) are deliberately not consulted: they say
@@ -83,14 +84,26 @@ func (i Identity) IsPrivileged() bool {
return true
}
switch i.SID {
case sidLocalSystem, sidLocalService, sidNetworkService:
if i.SID == sidLocalSystem {
return true
}
return slices.Contains(i.Groups, sidAdministrators)
}
// SameUser reports whether two identities are the same local principal. Only
// the account is compared: the group set and the elevation flag describe what a
// token may do, not who it belongs to. A SID on either side decides the
// comparison, so a Windows principal never matches a Unix one on the UID both
// happen to leave at zero. The zero Identity carries uid 0, so callers must
// establish that both identities are real before the answer means anything.
func (i Identity) SameUser(other Identity) bool {
if i.SID != "" || other.SID != "" {
return i.SID == other.SID
}
return i.UID == other.UID
}
// String renders the identity for audit logs and denial messages.
func (i Identity) String() string {
if i.IsWindows() {
+121
View File
@@ -0,0 +1,121 @@
package ipcauth
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestIdentitySameUser(t *testing.T) {
tests := []struct {
name string
a Identity
b Identity
want bool
}{
{
name: "same uid",
a: Identity{UID: 1000, GID: 1000},
b: Identity{UID: 1000, GID: 1000},
want: true,
},
{
name: "same uid, different gid and pid still the same user",
a: Identity{UID: 1000, GID: 1000, PID: 11},
b: Identity{UID: 1000, GID: 27, PID: 22},
want: true,
},
{
name: "different uid",
a: Identity{UID: 1000},
b: Identity{UID: 1001},
want: false,
},
{
name: "same sid",
a: Identity{SID: "S-1-5-21-1-2-3-1001"},
b: Identity{SID: "S-1-5-21-1-2-3-1001"},
want: true,
},
{
name: "same sid, elevation and groups differ",
a: Identity{SID: "S-1-5-21-1-2-3-1001", Elevated: true, Groups: []string{sidAdministrators}},
b: Identity{SID: "S-1-5-21-1-2-3-1001"},
want: true,
},
{
name: "different sid",
a: Identity{SID: "S-1-5-21-1-2-3-1001"},
b: Identity{SID: "S-1-5-21-1-2-3-1002"},
want: false,
},
{
name: "a windows principal is never a unix one",
a: Identity{SID: "S-1-5-18"},
b: Identity{UID: 0},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, tt.a.SameUser(tt.b))
assert.Equal(t, tt.want, tt.b.SameUser(tt.a), "SameUser must be symmetric")
})
}
}
func TestIdentityIsPrivileged(t *testing.T) {
tests := []struct {
name string
id Identity
want bool
}{
{
name: "Root",
id: Identity{UID: 0, GID: 0},
want: true,
},
{
name: "Non-root",
id: Identity{UID: 1000, GID: 1000},
want: false,
},
{
name: "Local system windows",
id: Identity{SID: sidLocalSystem},
want: true,
},
{
name: "Windows elevated",
id: Identity{SID: "S-1-5-21-1927267129-3959769253-3036563910-1001", Elevated: true},
want: true,
},
{
name: "Admin group windows",
id: Identity{SID: "S-1-5-21-1927267129-3959769253-3036563910-1001", Groups: []string{sidAdministrators}},
want: true,
},
{
name: "Regular user windows",
id: Identity{SID: "S-1-5-21-1927267129-3959769253-3036563910-1001"},
want: false,
},
{
name: "Network service windows",
id: Identity{SID: sidNetworkService},
want: false,
},
{
name: "Local service windows",
id: Identity{SID: sidLocalService},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, tt.id.IsPrivileged())
})
}
}
+9 -1
View File
@@ -45,7 +45,15 @@ func init() {
// matching there would let a non-elevated shell of an administrator account
// act as an administrator, which is the boundary the token check exists to
// keep.
selfMayDelegate = !id.IsPrivileged()
selfMayDelegate = mayDelegate(id)
}
// mayDelegate reports whether a daemon running as id may extend its authority to
// callers sharing its identity. The shared service accounts are excluded: their
// SID is held by unrelated services, so matching on it would grant them the
// daemon's authority.
func mayDelegate(id Identity) bool {
return !id.IsPrivileged() && id.SID != sidLocalService && id.SID != sidNetworkService
}
// IsDaemonSelf reports whether an identity is this very process. The JSON gateway
+26 -1
View File
@@ -98,7 +98,7 @@ func TestIsPrivilegedCaller_SelfRule(t *testing.T) {
t.Cleanup(func() { selfIdentity, selfKnown, selfMayDelegate = prevID, prevKnown, prevDelegate })
selfIdentity, selfKnown = tt.self, tt.selfKnown
selfMayDelegate = tt.selfKnown && !tt.self.IsPrivileged()
selfMayDelegate = tt.selfKnown && mayDelegate(tt.self)
if got := IsPrivilegedCaller(tt.caller); got != tt.want {
t.Fatalf("IsPrivilegedCaller(%v) with daemon %v = %t, want %t",
@@ -132,3 +132,28 @@ func TestIsPrivilegedCaller_ThisProcess(t *testing.T) {
t.Errorf("an unrelated identity %v was treated as privileged", other)
}
}
// The shared service accounts are held by unrelated services, so a daemon running
// as one of them must not extend its authority to every process with that SID.
func TestMayDelegate(t *testing.T) {
tests := []struct {
name string
self Identity
want bool
}{
{name: "unprivileged unix user", self: Identity{UID: 1000}, want: true},
{name: "root", self: Identity{UID: 0}, want: false},
{name: "unprivileged windows user", self: Identity{SID: "S-1-5-21-1-2-3-1001"}, want: true},
{name: "elevated windows user", self: Identity{SID: "S-1-5-21-1-2-3-1001", Elevated: true}, want: false},
{name: "local system", self: Identity{SID: sidLocalSystem}, want: false},
{name: "local service", self: Identity{SID: sidLocalService}, want: false},
{name: "network service", self: Identity{SID: sidNetworkService}, want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := mayDelegate(tt.self); got != tt.want {
t.Errorf("mayDelegate(%+v) = %v, want %v", tt.self, got, tt.want)
}
})
}
}
@@ -0,0 +1,274 @@
// Package localmetrics exposes client connection state as a local
// Prometheus /metrics endpoint.
package localmetrics
import (
"context"
"errors"
"net"
"net/http"
"net/netip"
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
dto "github.com/prometheus/client_model/go"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/internal/peer"
)
// DefaultListenAddress is used when local metrics are enabled without an explicit address.
const DefaultListenAddress = "127.0.0.1:9191"
const (
shutdownTimeout = 3 * time.Second
readHeaderTimeout = 5 * time.Second
readTimeout = 10 * time.Second
writeTimeout = 30 * time.Second
idleTimeout = time.Minute
)
// statusSource provides the connection state snapshots the collector reads on scrape.
type statusSource interface {
GetPeerStates() []peer.State
GetManagementState() peer.ManagementState
GetSignalState() peer.SignalState
}
// GathererProvider returns the current client metrics gatherer, or nil when
// no engine is running. It is called on every scrape.
type GathererProvider func() prometheus.Gatherer
// Manager runs the local /metrics HTTP endpoint according to the active
// client configuration. Reconcile is safe to call on every config change.
type Manager struct {
status statusSource
clientMetrics GathererProvider
mu sync.Mutex
srv *http.Server
addr string
}
// NewManager creates a manager that serves metrics from status and
// clientMetrics and shuts down when ctx is canceled.
func NewManager(ctx context.Context, status statusSource, clientMetrics GathererProvider) *Manager {
m := &Manager{status: status, clientMetrics: clientMetrics}
go func() {
<-ctx.Done()
m.Stop()
}()
return m
}
// Reconcile starts, stops, or restarts the metrics endpoint to match the
// desired state. An empty addr falls back to DefaultListenAddress.
func (m *Manager) Reconcile(enabled bool, addr string) {
if addr == "" {
addr = DefaultListenAddress
}
warnIfNotLoopback(addr)
m.mu.Lock()
defer m.mu.Unlock()
if !enabled {
m.stop()
return
}
if m.srv != nil && m.addr == addr {
return
}
m.stop()
registry := prometheus.NewRegistry()
registry.MustRegister(newCollector(m.status))
gatherers := prometheus.Gatherers{registry, prometheus.GathererFunc(func() ([]*dto.MetricFamily, error) {
if m.clientMetrics == nil {
return nil, nil
}
g := m.clientMetrics()
if g == nil {
return nil, nil
}
return g.Gather()
})}
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.HandlerFor(gatherers, promhttp.HandlerOpts{}))
srv := &http.Server{
Addr: addr,
Handler: mux,
ReadHeaderTimeout: readHeaderTimeout,
ReadTimeout: readTimeout,
WriteTimeout: writeTimeout,
IdleTimeout: idleTimeout,
}
m.srv = srv
m.addr = addr
log.Infof("serving local metrics on http://%s/metrics", addr)
go func() {
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Errorf("failed to serve local metrics on %s: %v", addr, err)
m.clear(srv)
}
}()
}
// clear drops the reference to srv so a later Reconcile with the same
// address restarts it. A newer server may already have replaced it, in
// which case the reference must stay.
func (m *Manager) clear(srv *http.Server) {
m.mu.Lock()
defer m.mu.Unlock()
if m.srv != srv {
return
}
m.srv = nil
m.addr = ""
}
// Stop shuts down the metrics endpoint if it is running.
func (m *Manager) Stop() {
m.mu.Lock()
defer m.mu.Unlock()
m.stop()
}
// stop shuts down the running server. Callers must hold m.mu.
func (m *Manager) stop() {
if m.srv == nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
defer cancel()
if err := m.srv.Shutdown(ctx); err != nil {
log.Debugf("failed to shut down local metrics server: %v", err)
}
m.srv = nil
m.addr = ""
}
// collector converts status recorder snapshots into Prometheus metrics at scrape time.
type collector struct {
status statusSource
managementConnected *prometheus.Desc
signalConnected *prometheus.Desc
peersTotal *prometheus.Desc
peersConnected *prometheus.Desc
peerLatency *prometheus.Desc
}
func newCollector(status statusSource) *collector {
return &collector{
status: status,
managementConnected: prometheus.NewDesc(
"netbird_management_connected",
"Whether the client is connected to the management service (1 connected, 0 disconnected).",
nil, nil,
),
signalConnected: prometheus.NewDesc(
"netbird_signal_connected",
"Whether the client is connected to the signal service (1 connected, 0 disconnected).",
nil, nil,
),
peersTotal: prometheus.NewDesc(
"netbird_peers",
"Number of peers known to this client.",
nil, nil,
),
peersConnected: prometheus.NewDesc(
"netbird_peers_connected",
"Number of connected peers by connection type.",
[]string{"connection_type"}, nil,
),
peerLatency: prometheus.NewDesc(
"netbird_peer_latency_seconds",
"Round-trip latency per directly connected peer; relayed connections have no latency measurement.",
[]string{"peer"}, nil,
),
}
}
// Describe implements prometheus.Collector.
func (c *collector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.managementConnected
ch <- c.signalConnected
ch <- c.peersTotal
ch <- c.peersConnected
ch <- c.peerLatency
}
// Collect implements prometheus.Collector.
func (c *collector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.managementConnected, prometheus.GaugeValue, boolToFloat(c.status.GetManagementState().Connected))
ch <- prometheus.MustNewConstMetric(c.signalConnected, prometheus.GaugeValue, boolToFloat(c.status.GetSignalState().Connected))
peers := c.status.GetPeerStates()
ch <- prometheus.MustNewConstMetric(c.peersTotal, prometheus.GaugeValue, float64(len(peers)))
var p2p, relayed float64
for _, p := range peers {
if p.ConnStatus != peer.StatusConnected {
continue
}
if p.Relayed {
relayed++
continue
}
p2p++
if latency := p.Latency.Seconds(); latency > 0 {
ch <- prometheus.MustNewConstMetric(c.peerLatency, prometheus.GaugeValue, latency, p.FQDN)
}
}
ch <- prometheus.MustNewConstMetric(c.peersConnected, prometheus.GaugeValue, p2p, "p2p")
ch <- prometheus.MustNewConstMetric(c.peersConnected, prometheus.GaugeValue, relayed, "relay")
}
func boolToFloat(b bool) float64 {
if b {
return 1
}
return 0
}
// IsLoopback reports whether addr binds the endpoint to the local host only.
// An empty address means DefaultListenAddress. It fails closed: an address
// that cannot be confirmed loopback, including an unparseable one, is not.
func IsLoopback(addr string) bool {
if addr == "" {
addr = DefaultListenAddress
}
host, _, err := net.SplitHostPort(addr)
if err != nil {
return false
}
if host == "localhost" {
return true
}
ip, err := netip.ParseAddr(host)
if err != nil {
return false
}
return ip.Unmap().IsLoopback()
}
// warnIfNotLoopback logs a warning when the listen address cannot be
// confirmed to be local-only, since the endpoint exposes peer and
// connectivity details without authentication.
func warnIfNotLoopback(addr string) {
if IsLoopback(addr) {
return
}
log.Warnf("local metrics endpoint listens on non-loopback address %s and is reachable from the network without authentication", addr)
}
@@ -0,0 +1,151 @@
package localmetrics
import (
"context"
"fmt"
"io"
"net"
"net/http"
"strings"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/netbirdio/netbird/client/internal/peer"
)
type stubStatus struct {
peers []peer.State
management peer.ManagementState
signal peer.SignalState
}
func (s *stubStatus) GetPeerStates() []peer.State { return s.peers }
func (s *stubStatus) GetManagementState() peer.ManagementState { return s.management }
func (s *stubStatus) GetSignalState() peer.SignalState { return s.signal }
func testStatus() *stubStatus {
return &stubStatus{
management: peer.ManagementState{Connected: true},
signal: peer.SignalState{Connected: true},
peers: []peer.State{
{FQDN: "peer-a.netbird.cloud", IP: "100.90.0.1", ConnStatus: peer.StatusConnected, Relayed: false, Latency: 12 * time.Millisecond},
{FQDN: "peer-b.netbird.cloud", IP: "100.90.0.2", ConnStatus: peer.StatusConnected, Relayed: false, Latency: 36 * time.Millisecond},
{FQDN: "peer-c.netbird.cloud", IP: "100.90.0.3", ConnStatus: peer.StatusConnected, Relayed: true},
{FQDN: "peer-d.netbird.cloud", IP: "100.90.0.4", ConnStatus: peer.StatusIdle},
},
}
}
func TestCollector(t *testing.T) {
c := newCollector(testStatus())
expected := `
# HELP netbird_management_connected Whether the client is connected to the management service (1 connected, 0 disconnected).
# TYPE netbird_management_connected gauge
netbird_management_connected 1
# HELP netbird_peer_latency_seconds Round-trip latency per directly connected peer; relayed connections have no latency measurement.
# TYPE netbird_peer_latency_seconds gauge
netbird_peer_latency_seconds{peer="peer-a.netbird.cloud"} 0.012
netbird_peer_latency_seconds{peer="peer-b.netbird.cloud"} 0.036
# HELP netbird_peers Number of peers known to this client.
# TYPE netbird_peers gauge
netbird_peers 4
# HELP netbird_peers_connected Number of connected peers by connection type.
# TYPE netbird_peers_connected gauge
netbird_peers_connected{connection_type="p2p"} 2
netbird_peers_connected{connection_type="relay"} 1
# HELP netbird_signal_connected Whether the client is connected to the signal service (1 connected, 0 disconnected).
# TYPE netbird_signal_connected gauge
netbird_signal_connected 1
`
require.NoError(t, testutil.CollectAndCompare(c, strings.NewReader(expected)))
}
func TestServe(t *testing.T) {
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err, "must find a free port")
addr := ln.Addr().String()
require.NoError(t, ln.Close())
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
m := NewManager(ctx, testStatus(), nil)
m.Reconcile(true, addr)
var body string
require.Eventually(t, func() bool {
resp, err := http.Get(fmt.Sprintf("http://%s/metrics", addr))
if err != nil {
return false
}
defer resp.Body.Close()
data, err := io.ReadAll(resp.Body)
if err != nil || resp.StatusCode != http.StatusOK {
return false
}
body = string(data)
return true
}, 2*time.Second, 50*time.Millisecond, "metrics endpoint should come up")
assert.Contains(t, body, "netbird_peers 4")
assert.Contains(t, body, `netbird_peers_connected{connection_type="relay"} 1`)
assert.Contains(t, body, `netbird_peer_latency_seconds{peer="peer-a.netbird.cloud"} 0.012`)
}
// A server that never came up must not be remembered, otherwise reconciling the
// same address again is a no-op and the endpoint never recovers.
func TestReconcileForgetsAFailedServer(t *testing.T) {
blocker, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err, "must find a free port")
t.Cleanup(func() { _ = blocker.Close() })
addr := blocker.Addr().String()
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
m := NewManager(ctx, testStatus(), nil)
m.Reconcile(true, addr)
require.Eventually(t, func() bool {
m.mu.Lock()
defer m.mu.Unlock()
return m.srv == nil && m.addr == ""
}, 2*time.Second, 20*time.Millisecond, "the failed server should be dropped")
require.NoError(t, blocker.Close())
m.Reconcile(true, addr)
require.Eventually(t, func() bool {
resp, err := http.Get(fmt.Sprintf("http://%s/metrics", addr))
if err != nil {
return false
}
defer resp.Body.Close()
return resp.StatusCode == http.StatusOK
}, 2*time.Second, 50*time.Millisecond, "reconciling the same address should retry the bind")
}
func TestIsLoopback(t *testing.T) {
tests := map[string]bool{
"": true,
"127.0.0.1:9191": true,
"127.9.9.9:9191": true,
"[::1]:9191": true,
"[::ffff:127.0.0.1]:9191": true,
"localhost:9191": true,
"0.0.0.0:9191": false,
"[::]:9191": false,
"192.168.1.10:9191": false,
"not-an-address": false,
"example.com:9191": false,
}
for addr, want := range tests {
t.Run(addr, func(t *testing.T) {
assert.Equal(t, want, IsLoopback(addr), "loopback verdict for %q", addr)
})
}
}
-15
View File
@@ -10,21 +10,6 @@ import (
mgmProto "github.com/netbirdio/netbird/shared/management/proto"
)
func convertToFirewallProtocol(protocol mgmProto.RuleProtocol) (firewallManager.Protocol, error) {
switch protocol {
case mgmProto.RuleProtocol_TCP:
return firewallManager.ProtocolTCP, nil
case mgmProto.RuleProtocol_UDP:
return firewallManager.ProtocolUDP, nil
case mgmProto.RuleProtocol_ICMP:
return firewallManager.ProtocolICMP, nil
case mgmProto.RuleProtocol_ALL:
return firewallManager.ProtocolALL, nil
default:
return "", fmt.Errorf("invalid protocol type: %s", protocol.String())
}
}
func convertPortInfo(portInfo *mgmProto.PortInfo) (*firewallManager.Port, error) {
if portInfo == nil {
return nil, errors.New("portInfo cannot be nil")
+3 -20
View File
@@ -45,30 +45,13 @@ func (m *influxDBMetrics) RecordConnectionStages(
isReconnection bool,
timestamps ConnectionStageTimestamps,
) {
var signalingReceivedToConnection, connectionToWgHandshake, totalDuration float64
if !timestamps.SignalingReceived.IsZero() && !timestamps.ConnectionReady.IsZero() {
signalingReceivedToConnection = timestamps.ConnectionReady.Sub(timestamps.SignalingReceived).Seconds()
}
if !timestamps.ConnectionReady.IsZero() && !timestamps.WgHandshakeSuccess.IsZero() {
connectionToWgHandshake = timestamps.WgHandshakeSuccess.Sub(timestamps.ConnectionReady).Seconds()
}
if !timestamps.SignalingReceived.IsZero() && !timestamps.WgHandshakeSuccess.IsZero() {
totalDuration = timestamps.WgHandshakeSuccess.Sub(timestamps.SignalingReceived).Seconds()
}
attemptType := "initial"
if isReconnection {
attemptType = "reconnection"
}
signalingReceivedToConnection, connectionToWgHandshake, totalDuration := timestamps.Durations()
connTypeStr := connectionType.String()
tags := fmt.Sprintf("deployment_type=%s,connection_type=%s,attempt_type=%s,version=%s,os=%s,arch=%s,peer_id=%s,connection_pair_id=%s",
agentInfo.DeploymentType.String(),
connTypeStr,
attemptType,
attemptType(isReconnection),
agentInfo.Version,
agentInfo.OS,
agentInfo.Arch,
@@ -94,7 +77,7 @@ func (m *influxDBMetrics) RecordConnectionStages(
m.trimLocked()
log.Tracef("peer connection metrics [%s, %s, %s]: signalingReceived→connection: %.3fs, connection→wg_handshake: %.3fs, total: %.3fs",
agentInfo.DeploymentType.String(), connTypeStr, attemptType, signalingReceivedToConnection, connectionToWgHandshake, totalDuration)
agentInfo.DeploymentType.String(), connTypeStr, attemptType(isReconnection), signalingReceivedToConnection, connectionToWgHandshake, totalDuration)
}
func (m *influxDBMetrics) RecordSyncDuration(_ context.Context, agentInfo AgentInfo, duration time.Duration) {
+14 -3
View File
@@ -32,13 +32,24 @@ Clients do not talk to InfluxDB directly. An ingest server sits between clients
```text
Client ──POST──▶ Ingest Server (:8087) ──▶ InfluxDB (internal)
│
├─ Checks the X-Peer-ID header format
├─ Validates line protocol
├─ Allowlists measurements, fields, and tags
├─ Rejects out-of-bound values
└─ Serves remote config at /config
```
- **No secret/token-based client auth** — the ingest server holds the InfluxDB token server-side. Clients must send a hashed peer ID via `X-Peer-ID` header.
- **Intentionally unauthenticated** — the endpoint receives obfuscated telemetry from
the peers of both cloud and self-hosted deployments. For a self-hosted peer there is
no shared trust anchor with this server, so there is nothing to authenticate against.
- **`X-Peer-ID` is a correlation tag, not a credential** — it carries the obfuscated
peer identifier so samples from one peer can be grouped. The server only checks that
the header is well-formed (16 hex chars); a malformed value is rejected with
`400 Bad Request`, not `401`. Any well-formed value is accepted by design, and the
header must not be relied on for access control. The header itself is not forwarded
to InfluxDB — the stored `peer_id` tag comes from the request body and is constrained
only by the tag allowlist and the maximum tag value length.
- **The InfluxDB token stays server-side** — clients never hold a write credential.
- **InfluxDB is not exposed** — only accessible within the docker network
- Source: `ingest/main.go`
@@ -61,7 +72,7 @@ Tags:
- `version`: NetBird version string
- `os`: Operating system (linux, darwin, windows, android, ios, etc.)
- `arch`: CPU architecture (amd64, arm64, etc.)
- `peer_id`: anonymised peer identifier (truncated SHA-256 of the WireGuard public key)
- `peer_id`: obfuscated peer identifier (truncated SHA-256 of the WireGuard public key)
- `connection_pair_id`: deterministic identifier for the peer pair, identical on both sides
**Note:** `SignalingReceived` is set when the first offer or answer arrives from the remote peer (in both initial and reconnection paths). It excludes the potentially unbounded wait for the remote peer to come online.
@@ -195,7 +206,7 @@ docker compose up -d
```
This starts:
- **Ingest server** on http://localhost:8087 — accepts client metrics (requires `X-Peer-ID` header, no secret/token auth)
- **Ingest server** on http://localhost:8087 — accepts client metrics (unauthenticated by design; expects a well-formed `X-Peer-ID` correlation tag)
- **InfluxDB** — internal only, not exposed to host
- **Grafana** on http://localhost:3001
+24 -5
View File
@@ -22,6 +22,11 @@ const (
maxDurationSeconds = 86400.0 // reject any duration field > 24 hours
peerIDLength = 16 // truncated SHA-256: 8 bytes = 16 hex chars
maxTagValueLength = 64 // reject tag values longer than this
readTimeout = 30 * time.Second // must fit reading a compressed body up to maxBodySize
writeTimeout = 60 * time.Second // must exceed the upstream client timeout below
idleTimeout = 120 * time.Second
readHeaderTimeout = 10 * time.Second
maxHeaderBytes = 1 << 20 // 1 MB
)
type measurementSpec struct {
@@ -124,8 +129,17 @@ func main() {
fmt.Fprint(w, "ok") //nolint:errcheck
})
srv := &http.Server{
Addr: listenAddr,
ReadTimeout: readTimeout,
ReadHeaderTimeout: readHeaderTimeout,
WriteTimeout: writeTimeout,
IdleTimeout: idleTimeout,
MaxHeaderBytes: maxHeaderBytes,
}
log.Printf("ingest server listening on %s, forwarding to %s", listenAddr, influxURL)
if err := http.ListenAndServe(listenAddr, nil); err != nil { //nolint:gosec
if err := srv.ListenAndServe(); err != nil {
log.Fatal(err)
}
}
@@ -137,8 +151,8 @@ func handleIngest(client *http.Client, influxURL, influxToken string) http.Handl
return
}
if err := validateAuth(r); err != nil {
http.Error(w, err.Error(), http.StatusUnauthorized)
if err := validatePeerIDFormat(r); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
@@ -187,8 +201,13 @@ func forwardToInflux(w http.ResponseWriter, r *http.Request, client *http.Client
io.Copy(w, resp.Body) //nolint:errcheck
}
// validateAuth checks that the X-Peer-ID header contains a valid hashed peer ID.
func validateAuth(r *http.Request) error {
// validatePeerIDFormat checks the shape of the X-Peer-ID header. The header is a
// correlation tag, not a credential: this endpoint is intentionally
// unauthenticated so that peers of self-hosted deployments, for which no shared
// trust anchor exists, can report obfuscated telemetry. The header is not forwarded
// to InfluxDB, so this check does not bound the stored peer_id tag; it only rejects
// a malformed header as a bad request rather than an auth failure.
func validatePeerIDFormat(r *http.Request) error {
peerID := r.Header.Get("X-Peer-ID")
if peerID == "" {
return fmt.Errorf("missing X-Peer-ID header")
@@ -94,7 +94,7 @@ func TestValidateLineProtocol_RejectsOnBadLine(t *testing.T) {
require.Error(t, err)
}
func TestValidateAuth(t *testing.T) {
func TestValidatePeerIDFormat(t *testing.T) {
tests := []struct {
name string
peerID string
@@ -113,7 +113,7 @@ func TestValidateAuth(t *testing.T) {
if tt.peerID != "" {
r.Header.Set("X-Peer-ID", tt.peerID)
}
err := validateAuth(r)
err := validatePeerIDFormat(r)
if tt.wantErr {
require.Error(t, err)
} else {
+23
View File
@@ -89,6 +89,21 @@ type ConnectionStageTimestamps struct {
WgHandshakeSuccess time.Time
}
// Durations returns the stage durations in seconds. A duration is zero when
// either of its timestamps is missing.
func (c ConnectionStageTimestamps) Durations() (signalingToConnection, connectionToWgHandshake, total float64) {
if !c.SignalingReceived.IsZero() && !c.ConnectionReady.IsZero() {
signalingToConnection = c.ConnectionReady.Sub(c.SignalingReceived).Seconds()
}
if !c.ConnectionReady.IsZero() && !c.WgHandshakeSuccess.IsZero() {
connectionToWgHandshake = c.WgHandshakeSuccess.Sub(c.ConnectionReady).Seconds()
}
if !c.SignalingReceived.IsZero() && !c.WgHandshakeSuccess.IsZero() {
total = c.WgHandshakeSuccess.Sub(c.SignalingReceived).Seconds()
}
return signalingToConnection, connectionToWgHandshake, total
}
// String returns a human-readable representation of the connection stage timestamps
func (c ConnectionStageTimestamps) String() string {
return fmt.Sprintf("ConnectionStageTimestamps{SignalingReceived=%v, ConnectionReady=%v, WgHandshakeSuccess=%v}",
@@ -279,3 +294,11 @@ func (c *ClientMetrics) stopPushLocked() {
c.wg.Wait()
c.push.Store(nil)
}
// attemptType returns the metric label for an initial vs reconnection attempt.
func attemptType(isReconnection bool) string {
if isReconnection {
return "reconnection"
}
return "initial"
}
+15 -1
View File
@@ -2,10 +2,24 @@
package metrics
import "github.com/prometheus/client_golang/prometheus"
// NewClientMetrics creates a new ClientMetrics instance
func NewClientMetrics(agentInfo AgentInfo) *ClientMetrics {
return &ClientMetrics{
impl: newInfluxDBMetrics(),
impl: newPrometheusMetrics(newInfluxDBMetrics()),
agentInfo: agentInfo,
}
}
// PrometheusGatherer returns the registry with the mirrored Prometheus
// metrics, or nil when unavailable.
func (c *ClientMetrics) PrometheusGatherer() prometheus.Gatherer {
if c == nil {
return nil
}
if pm, ok := c.impl.(*prometheusMetrics); ok {
return pm.Gatherer()
}
return nil
}
+119
View File
@@ -0,0 +1,119 @@
//go:build !js
package metrics
import (
"context"
"io"
"strconv"
"time"
"github.com/prometheus/client_golang/prometheus"
)
// prometheusMetrics mirrors recorded client metrics into a Prometheus
// registry for the local /metrics endpoint, then delegates to the wrapped
// implementation. Export and Reset pass through untouched: Prometheus
// metrics are cumulative and pull-based.
type prometheusMetrics struct {
next metricsImplementation
registry *prometheus.Registry
connectionStages *prometheus.HistogramVec
syncDuration prometheus.Histogram
syncPhaseDuration *prometheus.HistogramVec
loginDuration *prometheus.HistogramVec
}
func newPrometheusMetrics(next metricsImplementation) *prometheusMetrics {
connectionBuckets := []float64{.05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60}
m := &prometheusMetrics{
next: next,
registry: prometheus.NewRegistry(),
connectionStages: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "netbird_peer_connection_stage_duration_seconds",
Help: "Duration of peer connection establishment stages.",
Buckets: connectionBuckets,
}, []string{"stage", "connection_type", "attempt_type"}),
syncDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
Name: "netbird_sync_duration_seconds",
Help: "Duration of management sync message processing.",
Buckets: prometheus.DefBuckets,
}),
syncPhaseDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "netbird_sync_phase_duration_seconds",
Help: "Duration of individual sync processing phases.",
Buckets: prometheus.DefBuckets,
}, []string{"phase"}),
loginDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "netbird_login_duration_seconds",
Help: "Duration of logins to the management service.",
Buckets: prometheus.DefBuckets,
}, []string{"success"}),
}
m.registry.MustRegister(m.connectionStages, m.syncDuration, m.syncPhaseDuration, m.loginDuration)
return m
}
// Gatherer returns the registry holding the mirrored metrics.
func (m *prometheusMetrics) Gatherer() prometheus.Gatherer {
return m.registry
}
// RecordConnectionStages implements metricsImplementation.
func (m *prometheusMetrics) RecordConnectionStages(
ctx context.Context,
agentInfo AgentInfo,
connectionPairID string,
connectionType ConnectionType,
isReconnection bool,
timestamps ConnectionStageTimestamps,
) {
attempt := attemptType(isReconnection)
connType := connectionType.String()
signalingToConnection, connectionToWgHandshake, total := timestamps.Durations()
if signalingToConnection > 0 {
m.connectionStages.WithLabelValues("signaling_to_connection", connType, attempt).Observe(signalingToConnection)
}
if connectionToWgHandshake > 0 {
m.connectionStages.WithLabelValues("connection_to_wg_handshake", connType, attempt).Observe(connectionToWgHandshake)
}
if total > 0 {
m.connectionStages.WithLabelValues("total", connType, attempt).Observe(total)
}
m.next.RecordConnectionStages(ctx, agentInfo, connectionPairID, connectionType, isReconnection, timestamps)
}
// RecordSyncDuration implements metricsImplementation.
func (m *prometheusMetrics) RecordSyncDuration(ctx context.Context, agentInfo AgentInfo, duration time.Duration) {
m.syncDuration.Observe(duration.Seconds())
m.next.RecordSyncDuration(ctx, agentInfo, duration)
}
// RecordSyncPhase implements metricsImplementation.
func (m *prometheusMetrics) RecordSyncPhase(ctx context.Context, agentInfo AgentInfo, phase string, duration time.Duration) {
m.syncPhaseDuration.WithLabelValues(phase).Observe(duration.Seconds())
m.next.RecordSyncPhase(ctx, agentInfo, phase, duration)
}
// RecordLoginDuration implements metricsImplementation.
func (m *prometheusMetrics) RecordLoginDuration(ctx context.Context, agentInfo AgentInfo, duration time.Duration, success bool) {
m.loginDuration.WithLabelValues(strconv.FormatBool(success)).Observe(duration.Seconds())
m.next.RecordLoginDuration(ctx, agentInfo, duration, success)
}
// Export implements metricsImplementation by delegating to the wrapped
// implementation; Prometheus metrics are pulled via the registry instead.
func (m *prometheusMetrics) Export(w io.Writer) error {
return m.next.Export(w)
}
// Reset implements metricsImplementation by delegating to the wrapped
// implementation; Prometheus metrics must not be cleared on push.
func (m *prometheusMetrics) Reset() {
m.next.Reset()
}
+25 -9
View File
@@ -135,9 +135,10 @@ type Conn struct {
// used to store the remote Rosenpass key for Relayed connection in case of connection update from ice
rosenpassRemoteKey []byte
wgProxyICE wgproxy.Proxy
wgProxyRelay wgproxy.Proxy
handshaker *Handshaker
wgProxyICE wgproxy.Proxy
wgProxyRelay wgproxy.Proxy
relayedConnRef *relayClient.Conn
handshaker *Handshaker
guard *guard.Guard
wg sync.WaitGroup
@@ -560,7 +561,7 @@ func (conn *Conn) onRelayConnectionIsReady(rci RelayConnInfo) {
conn.mu.Lock()
defer conn.mu.Unlock()
if conn.ctx.Err() != nil {
if conn.ctx.Err() != nil || rci.relayedConn.Context().Err() != nil {
if err := rci.relayedConn.Close(); err != nil {
conn.Log.Warnf("failed to close unnecessary relayed connection: %v", err)
}
@@ -575,7 +576,9 @@ func (conn *Conn) onRelayConnectionIsReady(rci RelayConnInfo) {
conn.Log.Errorf("failed to add relayed net.Conn to local proxy: %v", err)
return
}
wgProxy.SetDisconnectListener(conn.onRelayDisconnected)
wgProxy.SetDisconnectListener(func() {
conn.onRelayDisconnected(rci.relayedConn)
})
conn.dumpState.NewLocalProxy()
@@ -583,7 +586,7 @@ func (conn *Conn) onRelayConnectionIsReady(rci RelayConnInfo) {
if conn.isICEActive() {
conn.Log.Debugf("do not switch to relay because current priority is: %s", conn.currentConnPriority.String())
conn.setRelayedProxy(wgProxy)
conn.setRelayedProxy(wgProxy, rci.relayedConn)
conn.statusRelay.SetConnected()
conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey, time.Now())
return
@@ -614,15 +617,26 @@ func (conn *Conn) onRelayConnectionIsReady(rci RelayConnInfo) {
conn.rosenpassRemoteKey = rci.rosenpassPubKey
conn.currentConnPriority = conntype.Relay
conn.statusRelay.SetConnected()
conn.setRelayedProxy(wgProxy)
conn.setRelayedProxy(wgProxy, rci.relayedConn)
conn.updateRelayStatus(rci.relayedConn.RemoteAddr().String(), rci.rosenpassPubKey, updateTime)
conn.Log.Infof("start to communicate with peer via relay")
conn.doOnConnected(rci.rosenpassPubKey, rci.rosenpassAddr, updateTime)
}
func (conn *Conn) onRelayDisconnected() {
// onRelayDisconnected reports the teardown of a relayed connection. relayedConn
// names the connection the signal belongs to, so a signal that arrives after
// its connection was replaced is ignored instead of tearing down its successor.
// A nil relayedConn means the caller does not track generations and the current
// connection is always torn down.
func (conn *Conn) onRelayDisconnected(relayedConn *relayClient.Conn) {
conn.mu.Lock()
defer conn.mu.Unlock()
if relayedConn != nil && conn.relayedConnRef != relayedConn {
conn.Log.Debugf("ignoring relay disconnect of a superseded connection")
return
}
conn.handleRelayDisconnectedLocked()
}
@@ -646,6 +660,7 @@ func (conn *Conn) handleRelayDisconnectedLocked() {
_ = conn.wgProxyRelay.CloseConn()
conn.wgProxyRelay = nil
}
conn.relayedConnRef = nil
changed := conn.statusRelay.Get() != worker.StatusDisconnected
if changed {
@@ -930,13 +945,14 @@ func (conn *Conn) logTraceConnState() {
}
}
func (conn *Conn) setRelayedProxy(proxy wgproxy.Proxy) {
func (conn *Conn) setRelayedProxy(proxy wgproxy.Proxy, relayedConn *relayClient.Conn) {
if conn.wgProxyRelay != nil {
if err := conn.wgProxyRelay.CloseConn(); err != nil {
conn.Log.Warnf("failed to close deprecated wg proxy conn: %v", err)
}
}
conn.wgProxyRelay = proxy
conn.relayedConnRef = relayedConn
}
// onWGHandshakeSuccess is called when the first WireGuard handshake is detected
+4 -4
View File
@@ -116,7 +116,7 @@ func (h *Handshaker) Listen(ctx context.Context) {
for {
select {
case remoteOfferAnswer := <-h.remoteOffersCh:
h.log.Infof("received offer, running version %s, remote WireGuard listen port %d, session id: %s, remote ICE supported: %t", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString(), remoteOfferAnswer.hasICECredentials())
h.log.Infof("received offer, running version %s, remote WireGuard listen port %d, session id: %s, remote ICE supported: %t, relay server: %s, relay IP: %s", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString(), remoteOfferAnswer.hasICECredentials(), remoteOfferAnswer.RelaySrvAddress, remoteOfferAnswer.RelaySrvIP)
// Record signaling received for reconnection attempts
if h.metricsStages != nil {
@@ -138,7 +138,7 @@ func (h *Handshaker) Listen(ctx context.Context) {
continue
}
case remoteOfferAnswer := <-h.remoteAnswerCh:
h.log.Infof("received answer, running version %s, remote WireGuard listen port %d, session id: %s, remote ICE supported: %t", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString(), remoteOfferAnswer.hasICECredentials())
h.log.Infof("received answer, running version %s, remote WireGuard listen port %d, session id: %s, remote ICE supported: %t, relay server: %s, relay IP: %s", remoteOfferAnswer.Version, remoteOfferAnswer.WgListenPort, remoteOfferAnswer.SessionIDString(), remoteOfferAnswer.hasICECredentials(), remoteOfferAnswer.RelaySrvAddress, remoteOfferAnswer.RelaySrvIP)
// Record signaling received for reconnection attempts
if h.metricsStages != nil {
@@ -209,14 +209,14 @@ func (h *Handshaker) sendOffer() error {
}
offer := h.buildOfferAnswer()
h.log.Debugf("sending offer with serial: %s", offer.SessionIDString())
h.log.Debugf("sending offer with serial: %s, relay server: %s, relay IP: %s", offer.SessionIDString(), offer.RelaySrvAddress, offer.RelaySrvIP)
return h.signaler.SignalOffer(offer, h.config.Key)
}
func (h *Handshaker) sendAnswer() error {
answer := h.buildOfferAnswer()
h.log.Debugf("sending answer with serial: %s", answer.SessionIDString())
h.log.Debugf("sending answer with serial: %s, relay server: %s, relay IP: %s", answer.SessionIDString(), answer.RelaySrvAddress, answer.RelaySrvIP)
return h.signaler.SignalAnswer(answer, h.config.Key)
}
+1 -4
View File
@@ -39,10 +39,7 @@ func NewAgent(ctx context.Context, iFaceDiscover stdnet.ExternalIFaceDiscover, c
iceFailedTimeout := iceFailedTimeout()
iceRelayAcceptanceMinWait := iceRelayAcceptanceMinWait()
transportNet, err := newStdNet(ctx, iFaceDiscover, config.InterfaceBlackList)
if err != nil {
log.Errorf("failed to create pion's stdnet: %s", err)
}
transportNet := newStdNet(ctx, iFaceDiscover, config.InterfaceBlackList)
fac := logging.NewDefaultLoggerFactory()
+1 -1
View File
@@ -8,6 +8,6 @@ import (
"github.com/netbirdio/netbird/client/internal/stdnet"
)
func newStdNet(ctx context.Context, _ stdnet.ExternalIFaceDiscover, ifaceBlacklist []string) (*stdnet.Net, error) {
func newStdNet(ctx context.Context, _ stdnet.ExternalIFaceDiscover, ifaceBlacklist []string) *stdnet.Net {
return stdnet.NewNet(ctx, ifaceBlacklist)
}
+1 -1
View File
@@ -6,6 +6,6 @@ import (
"github.com/netbirdio/netbird/client/internal/stdnet"
)
func newStdNet(ctx context.Context, iFaceDiscover stdnet.ExternalIFaceDiscover, ifaceBlacklist []string) (*stdnet.Net, error) {
func newStdNet(ctx context.Context, iFaceDiscover stdnet.ExternalIFaceDiscover, ifaceBlacklist []string) *stdnet.Net {
return stdnet.NewNetWithDiscover(ctx, iFaceDiscover, ifaceBlacklist)
}
+34 -2
View File
@@ -196,6 +196,7 @@ type Status struct {
muxRelays sync.RWMutex
peers map[string]State
ipToKey map[string]string
activeRoutePeers map[route.HAUniqueID]string
changeNotify map[string]map[string]*StatusChangeSubscription // map[peerID]map[subscriptionID]*StatusChangeSubscription
signalState bool
signalError error
@@ -257,6 +258,7 @@ func NewRecorder(mgmAddress string) *Status {
return &Status{
peers: make(map[string]State),
ipToKey: make(map[string]string),
activeRoutePeers: make(map[route.HAUniqueID]string),
changeNotify: make(map[string]map[string]*StatusChangeSubscription),
eventStreams: make(map[string]chan *proto.SystemEvent),
eventQueue: NewEventQueue(eventQueueSize),
@@ -481,6 +483,24 @@ func (d *Status) RemovePeerStateRoute(peer string, route string) error {
return nil
}
func (d *Status) AddActiveRoutePeer(haID route.HAUniqueID, peer string) {
d.mux.Lock()
defer d.mux.Unlock()
d.activeRoutePeers[haID] = peer
}
func (d *Status) RemoveActiveRoutePeer(haID route.HAUniqueID) {
d.mux.Lock()
defer d.mux.Unlock()
delete(d.activeRoutePeers, haID)
}
func (d *Status) GetActiveRoutePeers() map[route.HAUniqueID]string {
d.mux.RLock()
defer d.mux.RUnlock()
return maps.Clone(d.activeRoutePeers)
}
// CheckRoutes checks if the source and destination addresses are within the same route
// and returns the resource ID of the route that contains the addresses
func (d *Status) CheckRoutes(ip netip.Addr) ([]byte, bool) {
@@ -819,8 +839,8 @@ func (d *Status) SetSessionExpiresAt(deadline time.Time) {
// "none" would blank the UI at the exact moment it should say the session
// ended.
func (d *Status) GetSessionExpiresAt() time.Time {
d.mux.Lock()
defer d.mux.Unlock()
d.mux.RLock()
defer d.mux.RUnlock()
return d.sessionExpiresAt
}
@@ -1167,6 +1187,18 @@ func (d *Status) GetResolvedDomainsStates() map[domain.Domain]ResolvedDomainInfo
return maps.Clone(d.resolvedDomainsStates)
}
// GetPeerStates returns a snapshot of all known peer states, including offline peers.
func (d *Status) GetPeerStates() []State {
d.mux.RLock()
defer d.mux.RUnlock()
states := make([]State, 0, d.numOfPeers())
for _, state := range d.peers {
states = append(states, state)
}
return append(states, d.offlinePeers...)
}
// GetFullStatus gets full status
func (d *Status) GetFullStatus() FullStatus {
fullStatus := FullStatus{
+45
View File
@@ -9,6 +9,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/netbirdio/netbird/route"
)
func TestAddPeer(t *testing.T) {
@@ -129,6 +131,28 @@ func TestStatus_PeerStateByIP_RemovedPeer(t *testing.T) {
req.False(ok, "removed peer must not resolve by IPv6 tunnel address")
}
// TestStatus_GetPeerStates_IncludesOfflinePeers keeps the snapshot in line with
// GetFullStatus: offline peers are known peers, so a consumer counting peers
// must see the same total the status command reports.
func TestStatus_GetPeerStates_IncludesOfflinePeers(t *testing.T) {
status := NewRecorder("https://mgm")
req := require.New(t)
req.NoError(status.AddPeer("pk-online", "online.netbird", "100.64.0.10", "fd00::1"))
status.ReplaceOfflinePeers([]State{
{PubKey: "pk-offline", FQDN: "offline.netbird", IP: "100.64.0.20", ConnStatus: StatusIdle},
})
states := status.GetPeerStates()
req.Len(states, 2, "snapshot must carry both the online and the offline peer")
keys := make([]string, 0, len(states))
for _, s := range states {
keys = append(keys, s.PubKey)
}
req.ElementsMatch([]string{"pk-online", "pk-offline"}, keys, "snapshot must carry both peers")
}
func TestStatus_UpdatePeerFQDN(t *testing.T) {
key := "abc"
fqdn := "peer-a.netbird.local"
@@ -350,3 +374,24 @@ func TestMarkServerStateDoesNotNotifyWhenUnchanged(t *testing.T) {
status.MarkManagementDisconnected(err)
assert.False(t, notified(ch), "redundant disconnect should not notify")
}
func TestActiveRoutePeers(t *testing.T) {
status := NewRecorder("https://mgm")
netA := route.HAUniqueID("net-a-10.0.0.0/24")
netB := route.HAUniqueID("net-b-10.0.0.0/24")
status.AddActiveRoutePeer(netA, "peerA")
status.AddActiveRoutePeer(netB, "peerB")
active := status.GetActiveRoutePeers()
assert.Equal(t, "peerA", active[netA])
assert.Equal(t, "peerB", active[netB])
status.RemoveActiveRoutePeer(netA)
delete(active, netB)
active = status.GetActiveRoutePeers()
_, ok := active[netA]
assert.False(t, ok)
assert.Equal(t, "peerB", active[netB])
}
+91 -30
View File
@@ -64,6 +64,9 @@ type WorkerICE struct {
// portForwardAttempted tracks if we've already tried port forwarding this session
portForwardAttempted bool
// dialFunc, when non-nil, replaces agentDial in connect(). Only for tests.
dialFunc func(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *OfferAnswer) (net.Conn, error)
}
func NewWorkerICE(ctx context.Context, log *log.Entry, config ConnConfig, conn *Conn, signaler *Signaler, ifaceDiscover stdnet.ExternalIFaceDiscover, statusRecorder *Status, hasRelayOnLocally bool) (*WorkerICE, error) {
@@ -118,12 +121,9 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
}
}
sessionID, err := NewICESessionID()
if err != nil {
w.log.Errorf("failed to create new session ID: %s", err)
}
w.sessionID = sessionID
w.agent = nil
// Keep the ID already advertised to the remote. Answers do not get a
// reply, so changing it here makes the next offer restart both sides.
w.abandonNegotiation()
}
var preferredCandidateTypes []ice.CandidateType
@@ -151,7 +151,9 @@ func (w *WorkerICE) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
w.remoteSessionID = ""
}
go w.connect(dialerCtx, agent, remoteOfferAnswer)
// Capture the cancel func at spawn time: connect reads it from the argument
// instead of the field, which a newer OnNewOffer may already have replaced.
go w.connect(dialerCtx, dialerCancel, agent, remoteOfferAnswer)
}
// OnRemoteCandidate Handles ICE connection Candidate provided by the remote peer.
@@ -200,16 +202,19 @@ func (w *WorkerICE) Close() {
w.muxAgent.Lock()
defer w.muxAgent.Unlock()
if w.agent == nil {
return
if w.agent != nil || w.agentConnecting {
w.renewSessionID()
}
w.agentDialerCancel()
if err := w.agent.Close(); err != nil {
w.log.Warnf("failed to close ICE agent: %s", err)
if w.agent != nil {
w.agentDialerCancel()
if err := w.agent.Close(); err != nil {
w.log.Warnf("failed to close ICE agent: %s", err)
}
}
w.agent = nil
// Unconditional: a dial goroutine racing this Close skips its own cleanup
// (closeAgent finds a nil agent), so the flags must be dropped here too or
// the reconnection guard reads the stale state as Connected forever.
w.abandonNegotiation()
}
func (w *WorkerICE) reCreateAgent(dialerCancel context.CancelFunc, candidates []ice.CandidateType) (*icemaker.ThreadSafeAgent, error) {
@@ -247,31 +252,52 @@ func (w *WorkerICE) SessionID() ICESessionID {
// will block until connection succeeded
// but it won't release if ICE Agent went into Disconnected or Failed state,
// so we have to cancel it with the provided context once agent detected a broken connection
func (w *WorkerICE) connect(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *OfferAnswer) {
func (w *WorkerICE) connect(ctx context.Context, dialerCancel context.CancelFunc, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *OfferAnswer) {
w.log.Debugf("gather candidates")
if err := agent.GatherCandidates(); err != nil {
w.log.Warnf("failed to gather candidates: %s", err)
w.closeAgent(agent, w.agentDialerCancel)
w.closeAgent(agent, dialerCancel)
return
}
w.log.Debugf("agent dial")
remoteConn, err := w.agentDial(ctx, agent, remoteOfferAnswer)
dial := func(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *OfferAnswer) (net.Conn, error) {
return w.agentDial(ctx, agent, remoteOfferAnswer)
}
if w.dialFunc != nil {
dial = w.dialFunc
}
remoteConn, err := dial(ctx, agent, remoteOfferAnswer)
if err != nil {
w.log.Debugf("failed to dial the remote peer: %s", err)
w.closeAgent(agent, w.agentDialerCancel)
w.closeAgent(agent, dialerCancel)
return
}
w.log.Debugf("agent dial succeeded")
// A newer negotiation may have replaced our agent while agentDial was
// blocked. Drop the dead connection before running pair retrieval, port
// punching or candidate work against a closed agent. The commit-point
// check below still guards a replacement arriving after this point.
w.muxAgent.Lock()
stale := w.agent != agent
w.muxAgent.Unlock()
if stale {
if err := remoteConn.Close(); err != nil {
w.log.Warnf("failed to close stale ICE connection: %s", err)
}
w.log.Warnf("discarding connection from a stale ICE negotiation")
return
}
pair, err := agent.GetSelectedCandidatePair()
if err != nil {
w.closeAgent(agent, w.agentDialerCancel)
w.closeAgent(agent, dialerCancel)
return
}
if pair == nil {
w.log.Warnf("selected candidate pair is nil, cannot proceed")
w.closeAgent(agent, w.agentDialerCancel)
w.closeAgent(agent, dialerCancel)
return
}
@@ -301,11 +327,27 @@ func (w *WorkerICE) connect(ctx context.Context, agent *icemaker.ThreadSafeAgent
w.log.Infof("connection succeeded with offer session: %s", remoteOfferAnswer.SessionIDString())
w.muxAgent.Lock()
// Authoritative ownership guard: a negotiation that lost w.agent to a newer
// one between the post-dial check and the commit must not clear agentConnecting,
// record lastSuccess or report the connection, so the state commit has to be
// atomic with the check.
if w.agent != agent {
w.muxAgent.Unlock()
if err := remoteConn.Close(); err != nil {
w.log.Warnf("failed to close stale ICE connection: %s", err)
}
w.log.Warnf("discarding connection from a stale ICE negotiation")
return
}
w.agentConnecting = false
w.lastSuccess = time.Now()
w.muxAgent.Unlock()
// todo: the potential problem is a race between the onConnectionStateChange
// and the delivery below: after this unlock, a newer offer can replace
// w.agent before onICEConnectionIsReady runs, delivering this (now stale)
// connection. The newer negotiation overwrites it with its own delivery,
// so the window only ever downgrades an endpoint transiently.
w.conn.onICEConnectionIsReady(selectedPriority(pair), ci)
}
@@ -321,20 +363,39 @@ func (w *WorkerICE) closeAgent(agent *icemaker.ThreadSafeAgent, cancel context.C
sessionChanged := w.remoteSessionChanged
w.remoteSessionChanged = false
// Only the owner of the current session may reset its state: a stale dial
// goroutine waking after a newer attempt must not clobber it.
if w.agent == agent {
// consider to remove from here and move to the OnNewOffer
sessionID, err := NewICESessionID()
if err != nil {
w.log.Errorf("failed to create new session ID: %s", err)
}
w.sessionID = sessionID
w.agent = nil
w.agentConnecting = false
w.remoteSessionID = ""
w.renewSessionID()
w.abandonNegotiation()
}
return sessionChanged
}
// renewSessionID starts a new local session, so the remote treats our next offer
// or answer as a restart. Caller holds muxAgent.
func (w *WorkerICE) renewSessionID() {
sessionID, err := NewICESessionID()
if err != nil {
w.log.Errorf("failed to create new session ID: %s", err)
return
}
w.sessionID = sessionID
}
// abandonNegotiation drops all recorded ICE session state so the worker treats the
// next offer as a fresh start instead of a duplicate of a dead negotiation. The
// agent and agentConnecting flags must change together: leaving one stale wedges
// the reconnection guard into reporting Connected forever. It neither cancels an
// in-flight dial nor closes an agent — callers dispose of those themselves first,
// so a stale goroutine can never cancel another session's dial through this path.
// Caller must hold muxAgent.
func (w *WorkerICE) abandonNegotiation() {
w.agent = nil
w.agentConnecting = false
w.remoteSessionID = ""
}
func (w *WorkerICE) punchRemoteWGPort(pair *ice.CandidatePair, remoteWgPort int) {
// wait local endpoint configuration
time.Sleep(time.Second)
@@ -0,0 +1,257 @@
package peer
import (
"context"
"net"
"sync/atomic"
"testing"
"time"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
signal "github.com/netbirdio/netbird/shared/signal/client"
sProto "github.com/netbirdio/netbird/shared/signal/proto"
)
// stubSignalClient satisfies signal.Client as a no-op so the candidate
// goroutine spawned by a real GatherCandidates never dereferences a nil
// signaler in tests.
type stubSignalClient struct{}
func (stubSignalClient) Close() error { return nil }
func (stubSignalClient) StreamConnected() bool { return false }
func (stubSignalClient) GetStatus() signal.Status { return signal.StreamDisconnected }
func (stubSignalClient) Receive(context.Context, func(*sProto.Message) error) error { return nil }
func (stubSignalClient) Ready() bool { return false }
func (stubSignalClient) IsHealthy() bool { return false }
func (stubSignalClient) WaitStreamConnected(context.Context) {}
func (stubSignalClient) SendToStream(*sProto.EncryptedMessage) error { return nil }
func (stubSignalClient) Send(*sProto.Message) error { return nil }
func (stubSignalClient) SetOnReconnectedListener(func()) {}
// newTestWorkerICE builds a worker with real pion plumbing and no-op signaling.
func newTestWorkerICE(t *testing.T) *WorkerICE {
t.Helper()
config := connConf
stunTurn := &icemaker.StunTurn{}
stunTurn.Store(nil)
config.ICEConfig.StunTurn = stunTurn
w, err := NewWorkerICE(context.Background(), log.WithField("test", t.Name()), config, nil,
NewSignaler(stubSignalClient{}, wgtypes.Key{}), nil, nil, false)
require.NoError(t, err, "worker setup must succeed")
return w
}
// TestWorkerICE_CloseDuringDial_ClearsConnectingFlag drives the teardown race
// through the real dial goroutine instead of simulating its cleanup.
//
// The real-world sequence this models:
// 1. OnNewOffer starts a negotiation: agent set, agentConnecting = true,
// go connect()
// 2. The network dies and connect() stays blocked inside GatherCandidates/Dial
// 3. A WG handshake timeout calls Close(): the agent is released and the dial
// context cancelled, but agentConnecting is not reset
// 4. The real goroutine wakes with an error and runs its own cleanup
// (closeAgent), where `w.agent == agent` is now false, so the flag reset
// is skipped
//
// There is no remote responder, so Dial can never succeed: whatever point the
// goroutine is at, closing first forces it down the error path. Before the fix
// the flag stays true forever and the deadline below expires.
func TestWorkerICE_CloseDuringDial_ClearsConnectingFlag(t *testing.T) {
w := newTestWorkerICE(t)
sid := ICESessionID("test-session-id")
w.OnNewOffer(&OfferAnswer{
IceCredentials: IceCredentials{
UFrag: "testufrag",
Pwd: "testpwdtestpwdtestpwd12",
},
SessionID: &sid,
})
require.True(t, w.InProgress(), "OnNewOffer must mark the negotiation as in progress")
// Teardown wins the race while connect() is still running.
w.Close()
// Close drops the flags synchronously, so the assertion below does not
// converge on the goroutine: the deadline only absorbs the dial goroutine
// waking up in the background, proving nothing re-wedges it afterwards.
require.Eventually(t, func() bool {
return !w.InProgress()
}, 10*time.Second, 50*time.Millisecond,
"Close must leave the negotiation idle even while the dial goroutine is still winding down")
// abandonNegotiation owns these three fields together; the worker is idle
// only when all of them are dropped.
w.muxAgent.Lock()
defer w.muxAgent.Unlock()
assert.Nil(t, w.agent, "no agent may survive the teardown")
assert.False(t, w.agentConnecting, "the connecting flag must match the nil agent")
assert.Empty(t, w.remoteSessionID, "a dead session's remote ID must not linger")
}
// TestWorkerICE_CloseClearsResidualConnectingState covers Close on a worker whose
// agent is already gone but whose flag is stuck on true, e.g. after an aborted
// recreate in OnNewOffer or after a first Close raced a dial goroutine.
func TestWorkerICE_CloseClearsResidualConnectingState(t *testing.T) {
w := newTestWorkerICE(t)
w.muxAgent.Lock()
w.agentConnecting = true
w.muxAgent.Unlock()
w.Close()
assert.False(t, w.InProgress(), "Close must drop residual connecting state even without a live agent")
w.muxAgent.Lock()
defer w.muxAgent.Unlock()
assert.Nil(t, w.agent)
assert.False(t, w.agentConnecting)
assert.Empty(t, w.remoteSessionID)
}
// TestWorkerICE_StaleCloseAgentKeepsCurrentSession pins the ownership guard in
// closeAgent: a late-waking dial goroutine from an older session must not reset
// the state of a newer negotiation that reused the worker. The newer session
// must survive wholesale - agent, flag and remote session identity alike.
func TestWorkerICE_StaleCloseAgentKeepsCurrentSession(t *testing.T) {
w := newTestWorkerICE(t)
t.Cleanup(w.Close)
sidA := ICESessionID("session-a")
w.OnNewOffer(&OfferAnswer{
IceCredentials: IceCredentials{UFrag: "ufragaaaa", Pwd: "pwdpwdpwdpwdpwdpwdpwdp1"},
SessionID: &sidA,
})
w.muxAgent.Lock()
oldAgent := w.agent
oldCancel := w.agentDialerCancel
w.muxAgent.Unlock()
require.NotNil(t, oldAgent, "OnNewOffer must have created an ICE agent")
w.Close()
sidB := ICESessionID("session-b")
w.OnNewOffer(&OfferAnswer{
IceCredentials: IceCredentials{UFrag: "ufragbbbb", Pwd: "pwdpwdpwdpwdpwdpwdpwdp2"},
SessionID: &sidB,
})
require.True(t, w.InProgress(), "the second negotiation must be in flight")
w.muxAgent.Lock()
newAgent := w.agent
w.muxAgent.Unlock()
// The old dial goroutine finally wakes and cleans up its captured agent.
w.closeAgent(oldAgent, oldCancel)
w.muxAgent.Lock()
defer w.muxAgent.Unlock()
assert.Same(t, newAgent, w.agent, "the current agent must be untouched by the stale cleanup")
assert.True(t, w.agentConnecting, "the current negotiation must stay in flight")
// Read live under the lock: a snapshot captured before the stale cleanup
// would pass even if the cleanup wiped current state.
assert.Equal(t, sidB, w.remoteSessionID, "the remote session identity must be preserved")
}
// closeTrackConn records Close calls so a test can assert that a discarded
// connection was actually released.
type closeTrackConn struct {
net.Conn
closed atomic.Bool
}
func (c *closeTrackConn) Close() error {
c.closed.Store(true)
return c.Conn.Close()
}
// TestWorkerICE_StaleDialSuccessKeepsNewerNegotiation pins the ownership guard
// in connect()'s success path: a dial that came back after a newer negotiation
// replaced the agent must discard its connection and leave the newer session's
// state - agent, agentConnecting, remoteSessionID, lastSuccess - intact.
//
// The dial hook holds session A's goroutine open until session B is installed,
// then returns a live connection, mimicking the vendored pion dial which hands
// out a live *ice.Conn when a pair is selected without checking afterwards
// whether the agent was replaced meanwhile. Releasing A's dial therefore
// exercises the stale-success commit path deterministically instead of racing
// real ICE.
func TestWorkerICE_StaleDialSuccessKeepsNewerNegotiation(t *testing.T) {
w := newTestWorkerICE(t)
t.Cleanup(w.Close)
dialStarted := make(chan struct{})
releaseDial := make(chan struct{})
staleConn := &closeTrackConn{}
var calls atomic.Int32
w.dialFunc = func(ctx context.Context, _ *icemaker.ThreadSafeAgent, _ *OfferAnswer) (net.Conn, error) {
if calls.Add(1) == 1 {
// Session A: hold the goroutine open until session B is installed,
// then return a live connection, mimicking the vendored pion dial
// which hands out a live *ice.Conn once a pair is selected without
// re-checking whether the agent was replaced meanwhile. Releasing
// the dial therefore exercises the stale-success commit path
// deterministically instead of racing real ICE.
close(dialStarted)
<-releaseDial
client, _ := net.Pipe()
staleConn.Conn = client
return staleConn, nil
}
// A newer negotiation parks on its dialer context, cancelled by the
// t.Cleanup Close at test end.
<-ctx.Done()
return nil, ctx.Err()
}
sidA := ICESessionID("session-a")
w.OnNewOffer(&OfferAnswer{
IceCredentials: IceCredentials{UFrag: "ufragaaaa", Pwd: "pwdpwdpwdpwdpwdpwdpwdp1"},
SessionID: &sidA,
})
require.True(t, w.InProgress(), "session A must be in flight")
// Session A's goroutine is now parked in the dial hook.
<-dialStarted
sidB := ICESessionID("session-b")
w.OnNewOffer(&OfferAnswer{
IceCredentials: IceCredentials{UFrag: "ufragbbbb", Pwd: "pwdpwdpwdpwdpwdpwdpwdp2"},
SessionID: &sidB,
})
w.muxAgent.Lock()
agentB := w.agent
w.lastSuccess = time.Time{}
w.muxAgent.Unlock()
require.NotNil(t, agentB, "session B must have created an ICE agent")
require.True(t, w.InProgress(), "session B must be in flight")
// Release session A's dial: it must be recognized as stale and discarded.
close(releaseDial)
require.Eventually(t, func() bool {
return staleConn.closed.Load()
}, 10*time.Second, 10*time.Millisecond,
"the stale connection must be closed by the ownership guard")
w.muxAgent.Lock()
defer w.muxAgent.Unlock()
assert.Same(t, agentB, w.agent, "session A must not uninstall session B's agent")
assert.True(t, w.agentConnecting, "session A must not clear session B's connecting flag")
assert.Equal(t, sidB, w.remoteSessionID, "session A must not clear session B's remote session identity")
assert.True(t, w.lastSuccess.IsZero(), "session A must not record a success for session B")
// The commit block guards agentConnecting, lastSuccess and
// onICEConnectionIsReady together, so the state assertions above imply the
// callback never ran for session A; the nil conn would have panicked the
// stale goroutine on any invocation.
}
@@ -0,0 +1,375 @@
package peer
import (
"context"
"fmt"
"net"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
icemaker "github.com/netbirdio/netbird/client/internal/peer/ice"
)
func TestWorkerICE_RemoteRestartPreservesAdvertisedSession(t *testing.T) {
w := newTestWorkerICE(t)
t.Cleanup(w.Close)
w.dialFunc = parkDial
advertised := w.SessionID()
remoteSession := ICESessionID("remote-first")
offer := OfferAnswer{
IceCredentials: IceCredentials{UFrag: "remoteufrag", Pwd: "remote-password-long-enough"},
SessionID: &remoteSession,
}
w.OnNewOffer(&offer)
require.True(t, w.InProgress(), "the first remote session must start ICE")
w.muxAgent.Lock()
firstAgent := w.agent
w.muxAgent.Unlock()
// The same callback handles answers. A changed remote ID must not create
// an unannounced local ID that makes the remote restart on our next offer.
secondSession := ICESessionID("remote-restarted")
answer := offer
answer.SessionID = &secondSession
w.OnNewOffer(&answer)
assert.Equal(t, advertised, w.SessionID(), "following a remote restart must keep our advertised ID")
w.muxAgent.Lock()
secondAgent := w.agent
w.muxAgent.Unlock()
assert.NotSame(t, firstAgent, secondAgent, "the changed remote session must still rebuild ICE")
w.OnNewOffer(&answer)
w.muxAgent.Lock()
defer w.muxAgent.Unlock()
assert.Same(t, secondAgent, w.agent, "a repeated answer must keep the replacement agent")
}
func TestWorkerICE_LocalCloseChangesAdvertisedSession(t *testing.T) {
w := newTestWorkerICE(t)
dialStarted := make(chan struct{})
dialDone := make(chan struct{})
w.dialFunc = func(ctx context.Context, _ *icemaker.ThreadSafeAgent, _ *OfferAnswer) (net.Conn, error) {
close(dialStarted)
defer close(dialDone)
<-ctx.Done()
return nil, ctx.Err()
}
session := ICESessionID("remote-session")
w.OnNewOffer(&OfferAnswer{
IceCredentials: IceCredentials{UFrag: "remoteufrag", Pwd: "remote-password-long-enough"},
SessionID: &session,
})
<-dialStarted
advertised := w.SessionID()
w.Close()
assert.NotEqual(t, advertised, w.SessionID(), "a local teardown must tell the remote to restart")
closedSession := w.SessionID()
// The abandoned dial goroutine cleans up after Close returned.
<-dialDone
assert.Never(t, func() bool { return w.SessionID() != closedSession }, 200*time.Millisecond, 10*time.Millisecond,
"the late cleanup of a closed negotiation must not restart again")
w.Close()
assert.Equal(t, closedSession, w.SessionID(), "closing an idle worker must not restart again")
}
// parkDial stands in for the ICE dial. It never connects and returns once the
// negotiation is abandoned, so a test decides when a negotiation fails.
func parkDial(ctx context.Context, _ *icemaker.ThreadSafeAgent, _ *OfferAnswer) (net.Conn, error) {
<-ctx.Done()
return nil, ctx.Err()
}
func newTestSessionID(t *testing.T) ICESessionID {
t.Helper()
sid, err := NewICESessionID()
require.NoError(t, err)
return sid
}
// handshakeSide is one end of a simulated signaling exchange.
type handshakeSide interface {
// message builds the offer or answer the side would send now.
message() OfferAnswer
// receive hands a remote offer or answer to the side's ICE logic.
receive(msg OfferAnswer)
// teardowns counts negotiations the side tore down to follow a remote restart.
teardowns() int
// failAgent ends the side's current negotiation as an ICE failure does.
failAgent()
}
// workerSide drives a real WorkerICE.
type workerSide struct {
t *testing.T
w *WorkerICE
replaced int
}
func newWorkerSide(t *testing.T) *workerSide {
t.Helper()
w := newTestWorkerICE(t)
w.dialFunc = parkDial
t.Cleanup(w.Close)
return &workerSide{t: t, w: w}
}
func (s *workerSide) message() OfferAnswer {
sid := s.w.SessionID()
ufrag, pwd := s.w.GetLocalUserCredentials()
return OfferAnswer{IceCredentials: IceCredentials{UFrag: ufrag, Pwd: pwd}, SessionID: &sid}
}
func (s *workerSide) receive(msg OfferAnswer) {
before := s.agent()
s.w.OnNewOffer(&msg)
if after := s.agent(); before != nil && after != before {
s.replaced++
}
}
func (s *workerSide) teardowns() int { return s.replaced }
func (s *workerSide) agent() *icemaker.ThreadSafeAgent {
s.w.muxAgent.Lock()
defer s.w.muxAgent.Unlock()
return s.w.agent
}
// failAgent runs the cleanup the dial goroutine or the Failed state callback
// performs when the current negotiation dies.
func (s *workerSide) failAgent() {
s.t.Helper()
s.w.muxAgent.Lock()
agent, cancel := s.w.agent, s.w.agentDialerCancel
s.w.muxAgent.Unlock()
require.NotNil(s.t, agent, "failing requires a running negotiation")
s.w.closeAgent(agent, cancel)
}
// legacySide models a remote peer running a release from before this change:
// when it follows a remote restart it also picks a new session ID of its own,
// which it announces only with its next offer or answer.
type legacySide struct {
t *testing.T
sessionID ICESessionID
remoteID ICESessionID
hasAgent bool
replaced int
}
func newLegacySide(t *testing.T) *legacySide {
return &legacySide{t: t, sessionID: newTestSessionID(t)}
}
func (s *legacySide) message() OfferAnswer {
sid := s.sessionID
return OfferAnswer{
IceCredentials: IceCredentials{UFrag: "legacyufrag", Pwd: "legacy-password-long-enough"},
SessionID: &sid,
}
}
func (s *legacySide) receive(msg OfferAnswer) {
if msg.SessionID == nil {
s.hasAgent = true
return
}
if s.hasAgent {
if *msg.SessionID == s.remoteID {
return
}
s.replaced++
s.sessionID = newTestSessionID(s.t)
}
s.hasAgent = true
s.remoteID = *msg.SessionID
}
func (s *legacySide) teardowns() int { return s.replaced }
func (s *legacySide) failAgent() {
s.hasAgent = false
s.remoteID = ""
s.sessionID = newTestSessionID(s.t)
}
// exchange runs one guard-driven round in the order Handshaker.Listen uses: the
// answerer handles the offer and answers with the session ID it holds
// afterwards, and the offerer handles the answer without replying.
func exchange(offerer, answerer handshakeSide) {
answerer.receive(offerer.message())
offerer.receive(answerer.message())
}
// offerPattern decides which side's guard sends the offer in a round.
type offerPattern struct {
name string
picker func(round int, local, remote handshakeSide) (offerer, answerer handshakeSide)
}
var offerPatterns = []offerPattern{
{
// A routing peer whose relay is down keeps offering on its own.
name: "local peer offers",
picker: func(_ int, local, remote handshakeSide) (handshakeSide, handshakeSide) {
return local, remote
},
},
{
name: "both peers offer",
picker: func(round int, local, remote handshakeSide) (handshakeSide, handshakeSide) {
if round%2 == 0 {
return local, remote
}
return remote, local
},
},
}
// assertSettles runs guard rounds and requires the pair to stop restarting
// each other: at most maxTeardowns in total, and none once half the rounds ran.
func assertSettles(t *testing.T, pattern offerPattern, local, remote handshakeSide, maxTeardowns int) {
t.Helper()
const rounds = 10
total := func() int { return local.teardowns() + remote.teardowns() }
start := total()
var halfway int
for round := range rounds {
if round == rounds/2 {
halfway = total()
}
offerer, answerer := pattern.picker(round, local, remote)
exchange(offerer, answerer)
}
assert.LessOrEqual(t, total()-start, maxTeardowns, "the peers must not keep restarting each other")
assert.Equal(t, halfway, total(), "the negotiation must be stable in the later rounds")
}
// establish runs the first offer and answer, so both sides negotiate.
func establish(t *testing.T, local, remote handshakeSide) {
t.Helper()
exchange(local, remote)
require.Zero(t, local.teardowns()+remote.teardowns(), "the first exchange must not restart anything")
}
func TestICESession_SettlesAfterAgentFailure(t *testing.T) {
sides := []struct {
name string
remote func(t *testing.T) handshakeSide
}{
{name: "current remote", remote: func(t *testing.T) handshakeSide { return newWorkerSide(t) }},
{name: "legacy remote", remote: func(t *testing.T) handshakeSide { return newLegacySide(t) }},
}
failures := []struct {
name string
fail func(local, remote handshakeSide)
}{
{name: "remote agent fails", fail: func(_, remote handshakeSide) { remote.failAgent() }},
{name: "local agent fails", fail: func(local, _ handshakeSide) { local.failAgent() }},
{name: "both agents fail", fail: func(local, remote handshakeSide) {
local.failAgent()
remote.failAgent()
}},
}
for _, side := range sides {
for _, failure := range failures {
for _, pattern := range offerPatterns {
t.Run(fmt.Sprintf("%s/%s/%s", side.name, failure.name, pattern.name), func(t *testing.T) {
local := newWorkerSide(t)
remote := side.remote(t)
establish(t, local, remote)
failure.fail(local, remote)
assertSettles(t, pattern, local, remote, 2)
})
}
}
}
}
// TestICESession_LocalCloseRestartsRemote covers an explicit teardown, as on a
// WireGuard handshake timeout. The remote must start over as well, or it keeps
// answering from the negotiation this side just abandoned.
func TestICESession_LocalCloseRestartsRemote(t *testing.T) {
for _, pattern := range offerPatterns {
t.Run(pattern.name, func(t *testing.T) {
local := newWorkerSide(t)
remote := newWorkerSide(t)
establish(t, local, remote)
local.w.Close()
assertSettles(t, pattern, local, remote, 1)
assert.Equal(t, 1, remote.teardowns(), "the remote must restart its negotiation exactly once")
})
}
}
func TestICESession_DuplicateMessagesKeepNegotiation(t *testing.T) {
local := newWorkerSide(t)
remote := newWorkerSide(t)
offer := local.message()
remote.receive(offer)
answer := remote.message()
local.receive(answer)
// Signaling may deliver the same message again, and a peer answers every
// offer, including repeats of one it already handled.
remote.receive(offer)
local.receive(answer)
local.receive(remote.message())
assert.Zero(t, local.teardowns(), "a repeated answer must not restart the negotiation")
assert.Zero(t, remote.teardowns(), "a repeated offer must not restart the negotiation")
}
// TestICESession_RemoteWithoutSessionIDKeepsNegotiation covers remote peers
// too old to send session IDs: once negotiating, their messages cannot tell a
// restart from a repeat, so they must not tear anything down.
func TestICESession_RemoteWithoutSessionIDKeepsNegotiation(t *testing.T) {
local := newWorkerSide(t)
unversioned := OfferAnswer{IceCredentials: IceCredentials{UFrag: "oldufrag", Pwd: "old-password-long-enough"}}
local.receive(unversioned)
require.NotNil(t, local.agent(), "a message without a session ID must still start ICE")
advertised := local.w.SessionID()
for range 3 {
local.receive(unversioned)
}
assert.Zero(t, local.teardowns(), "messages without a session ID must not restart the negotiation")
assert.Equal(t, advertised, local.w.SessionID(), "the advertised session must not change")
}
// TestWorkerICE_StaleCleanupKeepsAdvertisedSession covers the cleanup of a
// replaced negotiation finishing late, from its dial goroutine or its Closed
// state callback. It must neither pick a new session ID, an unannounced local
// restart, nor disturb the negotiation that replaced it.
func TestWorkerICE_StaleCleanupKeepsAdvertisedSession(t *testing.T) {
local := newWorkerSide(t)
remote := newWorkerSide(t)
establish(t, local, remote)
local.w.muxAgent.Lock()
oldAgent, oldCancel := local.w.agent, local.w.agentDialerCancel
local.w.muxAgent.Unlock()
remote.failAgent()
exchange(local, remote)
require.Equal(t, 1, local.teardowns(), "the local side must follow the remote restart")
advertised := local.w.SessionID()
current := local.agent()
local.w.closeAgent(oldAgent, oldCancel)
assert.Equal(t, advertised, local.w.SessionID(), "a stale cleanup must not change the advertised session")
assert.Same(t, current, local.agent(), "a stale cleanup must keep the current negotiation")
assertSettles(t, offerPatterns[1], local, remote, 0)
}
+13 -14
View File
@@ -3,7 +3,6 @@ package peer
import (
"context"
"errors"
"net"
"net/netip"
"sync"
"sync/atomic"
@@ -14,7 +13,7 @@ import (
)
type RelayConnInfo struct {
relayedConn net.Conn
relayedConn *relayClient.Conn
rosenpassPubKey []byte
rosenpassAddr string
}
@@ -27,7 +26,7 @@ type WorkerRelay struct {
conn *Conn
relayManager *relayClient.Manager
relayedConn net.Conn
relayedConn *relayClient.Conn
relayLock sync.Mutex
relaySupportedOnRemotePeer atomic.Bool
@@ -80,12 +79,7 @@ func (w *WorkerRelay) OnNewOffer(remoteOfferAnswer *OfferAnswer) {
w.relayedConn = relayedConn
w.relayLock.Unlock()
err = w.relayManager.AddCloseListener(srv, w.onRelayClientDisconnected)
if err != nil {
log.Errorf("failed to add close listener: %s", err)
_ = relayedConn.Close()
return
}
go w.watchRelayedConn(relayedConn)
w.log.Debugf("peer conn opened via Relay: %s", srv)
go w.conn.onRelayConnectionIsReady(RelayConnInfo{
@@ -109,12 +103,15 @@ func (w *WorkerRelay) RelayIsSupportedLocally() bool {
func (w *WorkerRelay) CloseConn() {
w.relayLock.Lock()
defer w.relayLock.Unlock()
if w.relayedConn == nil {
conn := w.relayedConn
w.relayedConn = nil
w.relayLock.Unlock()
if conn == nil {
return
}
if err := w.relayedConn.Close(); err != nil {
if err := conn.Close(); err != nil {
w.log.Warnf("failed to close relay connection: %v", err)
}
}
@@ -133,6 +130,8 @@ func (w *WorkerRelay) preferredRelayServer(myRelayAddress, remoteRelayAddress st
return remoteRelayAddress
}
func (w *WorkerRelay) onRelayClientDisconnected() {
go w.conn.onRelayDisconnected()
func (w *WorkerRelay) watchRelayedConn(relayedConn *relayClient.Conn) {
<-relayedConn.Context().Done()
w.conn.onRelayDisconnected(relayedConn)
}
@@ -0,0 +1,76 @@
package profilemanager
import (
"fmt"
"sync"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Regression test: a concurrent Get and Set of the ActiveProfileState will
// fail on Windows since the write is a temp file renamed over an open file.
// Windows will refuse to replace a file another handle holds open by default.
func TestActiveProfileState_ReadsDoNotBreakAConcurrentWrite(t *testing.T) {
withTempConfigDir(t, func(configDir string) {
withPatchedGlobals(t, configDir, func() {
sm := &ServiceManager{}
require.NoError(t, sm.CreateDefaultProfile())
require.NoError(t, sm.SetActiveProfileStateToDefault())
const switched = ID("0123456789abcdef0123456789abcdef")
const rounds = 50
var wg sync.WaitGroup
errs := make(chan error, 128)
for i := 0; i < 8; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for r := 0; r < rounds; r++ {
state, err := sm.GetActiveProfileState()
if err != nil {
errs <- fmt.Errorf("read: %w", err)
return
}
if state.ID != defaultProfileName && state.ID != switched {
errs <- fmt.Errorf("read: active profile is %q, which no writer wrote", state.ID)
return
}
}
}()
}
for i := 0; i < 2; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for r := 0; r < rounds; r++ {
id := switched
if r%2 == 0 {
id = defaultProfileName
}
if err := sm.SetActiveProfileState(&ActiveProfileState{ID: id, Username: "testuser"}); err != nil {
errs <- fmt.Errorf("switch: %w", err)
return
}
}
}()
}
wg.Wait()
close(errs)
for err := range errs {
assert.NoError(t, err, "a switch and a read of the active profile state must not collide")
}
state, err := sm.GetActiveProfileState()
require.NoError(t, err)
assert.Contains(t, []ID{defaultProfileName, switched}, state.ID,
"the file holds whichever switch landed last, not a mix of the two")
})
})
}
+146 -13
View File
@@ -58,10 +58,6 @@ var DefaultInterfaceBlacklist = []string{
"Tailscale", "tailscale", "docker", "veth", "br-", "lo",
}
// loadMDMPolicy is the package-level indirection used by apply() to read the
// active MDM policy. Tests override this to inject a fake policy.
var loadMDMPolicy = mdm.LoadPolicy
// ConfigInput carries configuration changes to the client
type ConfigInput struct {
ManagementURL string
@@ -70,6 +66,7 @@ type ConfigInput struct {
StateFilePath string
PreSharedKey *string
ServerSSHAllowed *bool
RemoteJobsAllowed *bool
EnableSSHRoot *bool
EnableSSHSFTP *bool
EnableSSHLocalPortForwarding *bool
@@ -103,6 +100,9 @@ type ConfigInput struct {
DNSLabels domain.List
MTU *uint16
LocalMetricsEnabled *bool
LocalMetricsAddress *string
}
// Config Configuration type
@@ -124,6 +124,7 @@ type Config struct {
RosenpassEnabled bool
RosenpassPermissive bool
ServerSSHAllowed *bool
RemoteJobsAllowed *bool
EnableSSHRoot *bool
EnableSSHSFTP *bool
EnableSSHLocalPortForwarding *bool
@@ -144,6 +145,11 @@ type Config struct {
DNSLabels domain.List
// LocalMetricsEnabled enables the local Prometheus /metrics endpoint.
LocalMetricsEnabled bool
// LocalMetricsAddress is the listen address of the local /metrics endpoint.
LocalMetricsAddress string
// SSHKey is a private SSH key in a PEM format
SSHKey string
@@ -184,16 +190,34 @@ type Config struct {
// Runtime-only: re-derived from MDM policy on each load, never persisted.
LazyConnection string `json:"-"`
// DebugBundleUploadURL is the MDM-managed debug-bundle upload URL override.
// When set, it takes precedence over the management-supplied upload URL for
// remote debug bundle jobs. Runtime-only: re-derived from MDM policy on each
// load, never persisted.
DebugBundleUploadURL string `json:"-"`
MTU uint16
// policy is the MDM policy that produced the currently-set values for
// any MDM-enforced fields. Set by applyMDMPolicy at the tail of apply()
// and reset on every apply() invocation. Never persisted to disk.
// Callers query enforcement state via Policy() and the mdm.Policy API
// (HasKey, ManagedKeys, IsEmpty).
// policy is the MDM policy that produced the currently-set values
// for any MDM-enforced fields. Set by ApplyMDMPolicy on every
// invocation. Never persisted to disk. Callers query enforcement
// state via Policy() and the mdm.Policy API (HasKey, ManagedKeys,
// IsEmpty).
policy *mdm.Policy `json:"-"`
}
// ApplyMDMPolicy overlays the supplied MDM Policy on top of the current
// Config values and records it as Policy(). The overlay is not reversible:
// an empty Policy only clears the enforcement metadata, so resolve the base
// Config again (from disk or JSON) before applying a changed policy, the way
// the lifecycle owners do on every load.
func (config *Config) ApplyMDMPolicy(policy *mdm.Policy) {
if config == nil {
return
}
config.applyMDMPolicy(policy)
}
// Policy returns the MDM policy applied to this Config. Returns a non-nil
// empty Policy when MDM enforcement is inactive; callers can always invoke
// HasKey / ManagedKeys / IsEmpty without a nil check.
@@ -217,6 +241,12 @@ func getConfigDir() (string, error) {
}
configDir := filepath.Join(base, "netbird")
// Under sudo this is the invoking user's directory and strictly read-only:
// anything root creates in it would be root-owned and break the user's own
// runs. Reads of a missing directory fall through to defaults.
if sudoActive() {
return configDir, nil
}
if err := os.MkdirAll(configDir, 0o755); err != nil {
return "", err
}
@@ -224,6 +254,16 @@ func getConfigDir() (string, error) {
}
func baseConfigDir() (string, error) {
if u, ok := sudoInvokingUser(); ok {
return userBaseConfigDir(u)
}
// Fail closed instead of falling through to root's own config directory:
// reading root's active-profile and email state for what is actually the
// invoking user's invocation is the very confusion this resolution exists
// to prevent.
if sudoActive() {
return "", fmt.Errorf("resolve sudo invoking user %q: refusing to fall back to root's config directory", os.Getenv(envSudoUser))
}
if runtime.GOOS == "darwin" {
if u, err := user.Current(); err == nil && u.HomeDir != "" {
return filepath.Join(u.HomeDir, "Library", "Application Support"), nil
@@ -265,7 +305,10 @@ func createNewConfig(input ConfigInput) (*Config, error) {
config := &Config{
// defaults to false only for new (post 0.26) configurations
ServerSSHAllowed: util.False(),
WgPort: iface.DefaultWgPort,
// Remote jobs are an explicit opt-in and default off, including for
// legacy configs (a nil value materializes to false at connect time).
RemoteJobsAllowed: util.False(),
WgPort: iface.DefaultWgPort,
}
if _, err := config.apply(input); err != nil {
@@ -388,6 +431,18 @@ func (config *Config) apply(input ConfigInput) (updated bool, err error) {
updated = true
}
if input.LocalMetricsEnabled != nil && *input.LocalMetricsEnabled != config.LocalMetricsEnabled {
log.Infof("switching local metrics to %t", *input.LocalMetricsEnabled)
config.LocalMetricsEnabled = *input.LocalMetricsEnabled
updated = true
}
if input.LocalMetricsAddress != nil && *input.LocalMetricsAddress != config.LocalMetricsAddress {
log.Infof("switching local metrics address to %s", *input.LocalMetricsAddress)
config.LocalMetricsAddress = *input.LocalMetricsAddress
updated = true
}
if input.NetworkMonitor != nil && (config.NetworkMonitor == nil || *input.NetworkMonitor != *config.NetworkMonitor) {
log.Infof("switching Network Monitor to %t", *input.NetworkMonitor)
config.NetworkMonitor = input.NetworkMonitor
@@ -456,6 +511,21 @@ func (config *Config) apply(input ConfigInput) (updated bool, err error) {
updated = true
}
if input.RemoteJobsAllowed != nil && (config.RemoteJobsAllowed == nil || *input.RemoteJobsAllowed != *config.RemoteJobsAllowed) {
if *input.RemoteJobsAllowed {
log.Infof("enabling remote jobs")
} else {
log.Infof("disabling remote jobs")
}
config.RemoteJobsAllowed = input.RemoteJobsAllowed
updated = true
} else if config.RemoteJobsAllowed == nil {
// Remote jobs are an explicit opt-in: unlike SSH, a pre-existing config
// with no value defaults to disabled rather than being turned on.
config.RemoteJobsAllowed = util.False()
updated = true
}
if input.EnableSSHRoot != nil && (config.EnableSSHRoot == nil || *input.EnableSSHRoot != *config.EnableSSHRoot) {
if *input.EnableSSHRoot {
log.Infof("enabling SSH root login")
@@ -650,9 +720,11 @@ func (config *Config) apply(input ConfigInput) (updated bool, err error) {
updated = true
}
// MDM is the last override layer: any key present in the policy
// supersedes defaults, on-disk config, env vars and CLI input.
config.applyMDMPolicy(loadMDMPolicy())
// Initialise the MDM overlay to "no enforcement" so Config.Policy()
// never returns a stale or nil policy on a freshly applied Config.
// Lifecycle owners that want to enforce a real MDM policy invoke
// Config.ApplyMDMPolicy(loader.Load()) after this returns.
config.applyMDMPolicy(mdm.NewPolicy(nil))
return updated, nil
}
@@ -665,6 +737,14 @@ func (config *Config) apply(input ConfigInput) (updated bool, err error) {
// for the key, so per-field rejection of user writes still applies).
func (config *Config) applyMDMPolicy(policy *mdm.Policy) {
config.policy = policy
// DebugBundleUploadURL is a runtime-only override re-derived from MDM on
// every apply. Resolve it unconditionally (before the IsEmpty early return)
// so a policy that drops the key, becomes empty, or carries an invalid
// value can never leave a previously-enforced upload target active on a
// reused Config instance.
config.DebugBundleUploadURL = mdmDebugBundleUploadURL(policy)
if policy.IsEmpty() {
return
}
@@ -712,12 +792,19 @@ func (config *Config) applyMDMPolicy(policy *mdm.Policy) {
}
applyBool(mdm.KeyAllowServerSSH, func(v bool) { bv := v; config.ServerSSHAllowed = &bv })
applyBool(mdm.KeyRemoteJobsAllowed, func(v bool) { bv := v; config.RemoteJobsAllowed = &bv })
applyBool(mdm.KeyDisableClientRoutes, func(v bool) { config.DisableClientRoutes = v })
applyBool(mdm.KeyDisableServerRoutes, func(v bool) { config.DisableServerRoutes = v })
applyBool(mdm.KeyBlockInbound, func(v bool) { config.BlockInbound = v })
applyBool(mdm.KeyDisableAutoConnect, func(v bool) { config.DisableAutoConnect = v })
applyBool(mdm.KeyRosenpassEnabled, func(v bool) { config.RosenpassEnabled = v })
applyBool(mdm.KeyRosenpassPermissive, func(v bool) { config.RosenpassPermissive = v })
applyBool(mdm.KeyEnableLocalMetrics, func(v bool) { config.LocalMetricsEnabled = v })
if v, ok := policy.GetString(mdm.KeyLocalMetricsAddress); ok {
config.LocalMetricsAddress = v
logApplied(mdm.KeyLocalMetricsAddress, v)
}
if v, ok := policy.GetInt(mdm.KeyWireguardPort); ok {
// REG_DWORD is 32-bit; UDP port range is 1-65535. Clamp at the
@@ -739,6 +826,52 @@ func (config *Config) applyMDMPolicy(policy *mdm.Policy) {
config.LazyConnection = state
logApplied(mdm.KeyLazyConnection, state)
}
}
// ValidateBundleUploadURL sanity-checks a debug-bundle upload URL. An empty
// value is accepted — the executor falls back to the default upload service. A
// non-empty value must be a well-formed https URL with a host; a malformed
// value or a plaintext scheme is rejected. It deliberately does not constrain
// which host may receive the bundle. This is the single source of truth for the
// rule, shared by the remote-job executor (client/internal) and the MDM policy
// override below so the two validation paths cannot drift.
func ValidateBundleUploadURL(raw string) error {
if raw == "" {
return nil
}
parsed, err := url.Parse(raw)
if err != nil {
return fmt.Errorf("parse upload URL: %w", err)
}
// Hostname(), not Host: an authority like ":443" is non-empty but has no
// host, and would fail the actual upload.
if parsed.Scheme != "https" || parsed.Hostname() == "" {
return fmt.Errorf("upload URL must be an https URL with a host")
}
return nil
}
// mdmDebugBundleUploadURL resolves the MDM-enforced debug-bundle upload URL
// override from the policy, returning the empty string when the policy does
// not carry a valid KeyBundleUploadURL. An absent or invalid value fails
// closed to "" so it falls back to the management-supplied or default upload
// target rather than a previously-enforced one. The URL is never logged: it
// can embed credentials or signed query tokens (KeyBundleUploadURL is in
// mdm.SecretKeys).
func mdmDebugBundleUploadURL(policy *mdm.Policy) string {
v, ok := policy.GetString(mdm.KeyBundleUploadURL)
if !ok || v == "" {
return ""
}
// Must be a well-formed https URL with a host, matching the client's
// remote-job upload-URL validation (shared validator, single source of truth).
if err := ValidateBundleUploadURL(v); err != nil {
log.Warnf("MDM debug bundle upload URL is invalid (must be an https URL with a host); ignoring the override")
return ""
}
log.Infof("MDM override %s = ********** (secret)", mdm.KeyBundleUploadURL)
return v
}
// parseURL parses and validates the URL for the named service. The URL
@@ -0,0 +1,52 @@
package profilemanager
import (
"errors"
"fmt"
"github.com/netbirdio/netbird/client/mdm"
)
// ErrMDMManagedFields marks a config change rejected because it diverges from
// MDM-enforced values.
var ErrMDMManagedFields = errors.New("fields managed by MDM cannot be modified")
// MDMConflicts returns the names of MDM-managed keys whose requested value in
// the ConfigInput differs from the policy-enforced value; a field set to the
// enforced value is a no-op echo, not a conflict.
func MDMConflicts(input ConfigInput, policy *mdm.Policy) []string {
pskGot := input.PreSharedKey
if isPreSharedKeyHidden(pskGot) {
pskGot = nil
}
var port *int64
if input.WireguardPort != nil {
v := int64(*input.WireguardPort)
port = &v
}
return mdm.ResolveConflicts(policy, []mdm.ConflictCheck{
mdm.ConflictURL(mdm.KeyManagementURL, input.ManagementURL),
mdm.ConflictStringPtr(mdm.KeyPreSharedKey, pskGot),
mdm.ConflictBool(mdm.KeyRosenpassEnabled, input.RosenpassEnabled),
mdm.ConflictBool(mdm.KeyRosenpassPermissive, input.RosenpassPermissive),
mdm.ConflictBool(mdm.KeyDisableAutoConnect, input.DisableAutoConnect),
mdm.ConflictBool(mdm.KeyAllowServerSSH, input.ServerSSHAllowed),
mdm.ConflictBool(mdm.KeyRemoteJobsAllowed, input.RemoteJobsAllowed),
mdm.ConflictBool(mdm.KeyDisableClientRoutes, input.DisableClientRoutes),
mdm.ConflictBool(mdm.KeyDisableServerRoutes, input.DisableServerRoutes),
mdm.ConflictBool(mdm.KeyBlockInbound, input.BlockInbound),
mdm.ConflictInt64(mdm.KeyWireguardPort, port),
mdm.ConflictBool(mdm.KeyEnableLocalMetrics, input.LocalMetricsEnabled),
mdm.ConflictStringPtr(mdm.KeyLocalMetricsAddress, input.LocalMetricsAddress),
})
}
// CheckMDMConflicts returns an ErrMDMManagedFields-wrapped error naming the
// conflicting keys, or nil when the input does not fight the policy.
func CheckMDMConflicts(input ConfigInput, policy *mdm.Policy) error {
conflicts := MDMConflicts(input, policy)
if len(conflicts) == 0 {
return nil
}
return fmt.Errorf("%w: %v", ErrMDMManagedFields, conflicts)
}
+158 -59
View File
@@ -10,24 +10,58 @@ import (
"github.com/netbirdio/netbird/client/mdm"
)
// withMDMPolicy temporarily overrides the package-level loadMDMPolicy hook so
// apply() observes the supplied Policy. The original loader is restored at
// test cleanup.
func withMDMPolicy(t *testing.T, policy *mdm.Policy) {
// fakeFetcher implements mdm.PolicyFetcher returning a pre-set policy
// map. Test helper used to construct a Loader without touching the OS
// or any package-level state.
type fakeFetcher struct{ values map[string]any }
func (f *fakeFetcher) Fetch() map[string]any { return f.values }
// loaderFor builds an mdm.Loader whose loadPlatform returns the
// supplied Policy's underlying values.
func loaderFor(policy *mdm.Policy) *mdm.Loader {
if policy == nil || policy.IsEmpty() {
return mdm.NewLoader(&fakeFetcher{values: nil})
}
values := make(map[string]any)
for _, k := range policy.ManagedKeys() {
if v, ok := policy.GetString(k); ok {
values[k] = v
continue
}
if v, ok := policy.GetInt(k); ok {
values[k] = v
continue
}
if v, ok := policy.GetBool(k); ok {
values[k] = v
continue
}
if v, ok := policy.GetStringSlice(k); ok {
values[k] = v
}
}
return mdm.NewLoader(&fakeFetcher{values: values})
}
// configWithMDM is the test convenience that builds a Config via
// UpdateOrCreateConfig and overlays the supplied MDM policy on top —
// mirrors the production pattern (Server.getConfig / Client.applyMDMOverlay)
// where the Loader lives outside Config and the apply step is driven
// by the lifecycle owner.
func configWithMDM(t *testing.T, input ConfigInput, policy *mdm.Policy) *Config {
t.Helper()
prev := loadMDMPolicy
loadMDMPolicy = func() *mdm.Policy { return policy }
t.Cleanup(func() { loadMDMPolicy = prev })
cfg, err := UpdateOrCreateConfig(input)
require.NoError(t, err)
require.NotNil(t, cfg)
cfg.ApplyMDMPolicy(loaderFor(policy).Load())
return cfg
}
func TestApply_MDMEmpty_NoEnforcement(t *testing.T) {
withMDMPolicy(t, mdm.NewPolicy(nil))
cfg, err := UpdateOrCreateConfig(ConfigInput{
cfg := configWithMDM(t, ConfigInput{
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
})
require.NoError(t, err)
require.NotNil(t, cfg)
}, mdm.NewPolicy(nil))
assert.True(t, cfg.Policy().IsEmpty(), "no MDM source ⇒ empty Policy")
assert.False(t, cfg.Policy().HasKey(mdm.KeyManagementURL))
@@ -39,18 +73,15 @@ func TestApply_MDMEmpty_NoEnforcement(t *testing.T) {
func TestApply_MDMOnly_OverridesDefaults(t *testing.T) {
const mdmURL = "https://corp.mdm.example.com:443"
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
cfg := configWithMDM(t, ConfigInput{
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
}, mdm.NewPolicy(map[string]any{
mdm.KeyManagementURL: mdmURL,
mdm.KeyDisableClientRoutes: true,
mdm.KeyBlockInbound: true,
}))
cfg, err := UpdateOrCreateConfig(ConfigInput{
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
})
require.NoError(t, err)
require.NotNil(t, cfg)
assert.Equal(t, mdmURL, cfg.ManagementURL.String())
assert.True(t, cfg.DisableClientRoutes)
assert.True(t, cfg.BlockInbound)
@@ -65,16 +96,12 @@ func TestApply_MDMBeatsCLIInput(t *testing.T) {
const mdmURL = "https://mdm.example.com:443"
const cliURL = "https://cli.example.com:443"
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
mdm.KeyManagementURL: mdmURL,
}))
cfg, err := UpdateOrCreateConfig(ConfigInput{
cfg := configWithMDM(t, ConfigInput{
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
ManagementURL: cliURL,
})
require.NoError(t, err)
require.NotNil(t, cfg)
}, mdm.NewPolicy(map[string]any{
mdm.KeyManagementURL: mdmURL,
}))
// MDM wins over CLI-supplied management URL.
assert.Equal(t, mdmURL, cfg.ManagementURL.String())
@@ -82,16 +109,12 @@ func TestApply_MDMBeatsCLIInput(t *testing.T) {
}
func TestApply_MDMInvalidURL_KeepsPreviousValue(t *testing.T) {
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
cfg := configWithMDM(t, ConfigInput{
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
}, mdm.NewPolicy(map[string]any{
mdm.KeyManagementURL: "not-a-url",
}))
cfg, err := UpdateOrCreateConfig(ConfigInput{
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
})
require.NoError(t, err)
require.NotNil(t, cfg)
// Invalid MDM URL is logged and skipped: default URL stays in place
// to keep the client functional.
assert.Equal(t, DefaultManagementURL, cfg.ManagementURL.String())
@@ -106,30 +129,49 @@ func TestApply_MDMBoolKeysOverrideOnDiskValue(t *testing.T) {
tmp := filepath.Join(t.TempDir(), "config.json")
// Seed without MDM.
withMDMPolicy(t, mdm.NewPolicy(nil))
_, err := UpdateOrCreateConfig(ConfigInput{
configWithMDM(t, ConfigInput{
ConfigPath: tmp,
DisableClientRoutes: boolPtr(false),
RosenpassEnabled: boolPtr(false),
})
require.NoError(t, err)
}, mdm.NewPolicy(nil))
// Now enable MDM enforcement for these keys.
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
cfg := configWithMDM(t, ConfigInput{
ConfigPath: tmp,
}, mdm.NewPolicy(map[string]any{
mdm.KeyDisableClientRoutes: true,
mdm.KeyRosenpassEnabled: true,
}))
cfg, err := UpdateOrCreateConfig(ConfigInput{ConfigPath: tmp})
require.NoError(t, err)
require.NotNil(t, cfg)
assert.True(t, cfg.DisableClientRoutes, "MDM override should flip on-disk false to true")
assert.True(t, cfg.RosenpassEnabled)
assert.True(t, cfg.Policy().HasKey(mdm.KeyDisableClientRoutes))
assert.True(t, cfg.Policy().HasKey(mdm.KeyRosenpassEnabled))
}
func TestApply_MDMLocalMetrics(t *testing.T) {
tmp := filepath.Join(t.TempDir(), "config.json")
// Seed without MDM.
configWithMDM(t, ConfigInput{
ConfigPath: tmp,
LocalMetricsEnabled: boolPtr(false),
}, mdm.NewPolicy(nil))
// Now enable MDM enforcement for these keys.
cfg := configWithMDM(t, ConfigInput{
ConfigPath: tmp,
}, mdm.NewPolicy(map[string]any{
mdm.KeyEnableLocalMetrics: true,
mdm.KeyLocalMetricsAddress: "127.0.0.1:9292",
}))
assert.True(t, cfg.LocalMetricsEnabled, "MDM override should flip on-disk false to true")
assert.Equal(t, "127.0.0.1:9292", cfg.LocalMetricsAddress)
assert.True(t, cfg.Policy().HasKey(mdm.KeyEnableLocalMetrics))
assert.True(t, cfg.Policy().HasKey(mdm.KeyLocalMetricsAddress))
}
func TestApply_MDMLazyConnection(t *testing.T) {
cases := []struct {
name string
@@ -145,16 +187,12 @@ func TestApply_MDMLazyConnection(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
cfg := configWithMDM(t, ConfigInput{
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
}, mdm.NewPolicy(map[string]any{
mdm.KeyLazyConnection: c.raw,
}))
cfg, err := UpdateOrCreateConfig(ConfigInput{
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
})
require.NoError(t, err)
require.NotNil(t, cfg)
assert.Equal(t, c.want, cfg.LazyConnection)
assert.True(t, cfg.Policy().HasKey(mdm.KeyLazyConnection))
})
@@ -162,22 +200,83 @@ func TestApply_MDMLazyConnection(t *testing.T) {
}
func TestApply_MDMPreSharedKeyRedactionSentinelRejected(t *testing.T) {
const maskSentinel = "**********"
const maskSentinel = mdm.PreSharedKeyRedactedSentinel
withMDMPolicy(t, mdm.NewPolicy(map[string]any{
cfg := configWithMDM(t, ConfigInput{
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
}, mdm.NewPolicy(map[string]any{
mdm.KeyPreSharedKey: maskSentinel,
}))
cfg, err := UpdateOrCreateConfig(ConfigInput{
ConfigPath: filepath.Join(t.TempDir(), "config.json"),
})
require.NoError(t, err)
require.NotNil(t, cfg)
// Mask sentinel must not be persisted as the actual PSK.
assert.NotEqual(t, maskSentinel, cfg.PreSharedKey)
// Key still marked managed so user writes are still rejected.
assert.True(t, cfg.Policy().HasKey(mdm.KeyPreSharedKey))
}
func TestMDMConflicts_PreSharedKey(t *testing.T) {
policy := mdm.NewPolicy(map[string]any{
mdm.KeyPreSharedKey: "mdm-enforced-psk",
})
empty := ""
sentinel := mdm.PreSharedKeyRedactedSentinel
same := "mdm-enforced-psk"
other := "user-psk"
tests := []struct {
name string
psk *string
want []string
}{
{name: "unset", psk: nil, want: nil},
{name: "explicit empty", psk: &empty, want: []string{mdm.KeyPreSharedKey}},
{name: "sentinel echo", psk: &sentinel, want: nil},
{name: "same value", psk: &same, want: nil},
{name: "divergent", psk: &other, want: []string{mdm.KeyPreSharedKey}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.want, MDMConflicts(ConfigInput{PreSharedKey: tc.psk}, policy))
})
}
}
func TestMDMConflicts_RemoteJobsAndLocalMetrics(t *testing.T) {
policy := mdm.NewPolicy(map[string]any{
mdm.KeyRemoteJobsAllowed: false,
mdm.KeyEnableLocalMetrics: true,
mdm.KeyLocalMetricsAddress: "127.0.0.1:9999",
})
sameAddr := "127.0.0.1:9999"
otherAddr := "0.0.0.0:9999"
emptyAddr := ""
tests := []struct {
name string
input ConfigInput
want []string
}{
{name: "unset", input: ConfigInput{}, want: nil},
{name: "echo", input: ConfigInput{
RemoteJobsAllowed: boolPtr(false),
LocalMetricsEnabled: boolPtr(true),
LocalMetricsAddress: &sameAddr,
}, want: nil},
{name: "remote jobs divergent", input: ConfigInput{RemoteJobsAllowed: boolPtr(true)}, want: []string{mdm.KeyRemoteJobsAllowed}},
{name: "metrics disabled", input: ConfigInput{LocalMetricsEnabled: boolPtr(false)}, want: []string{mdm.KeyEnableLocalMetrics}},
{name: "metrics address divergent", input: ConfigInput{LocalMetricsAddress: &otherAddr}, want: []string{mdm.KeyLocalMetricsAddress}},
{name: "metrics address explicit empty", input: ConfigInput{LocalMetricsAddress: &emptyAddr}, want: []string{mdm.KeyLocalMetricsAddress}},
{name: "all divergent", input: ConfigInput{
RemoteJobsAllowed: boolPtr(true),
LocalMetricsEnabled: boolPtr(false),
LocalMetricsAddress: &otherAddr,
}, want: []string{mdm.KeyRemoteJobsAllowed, mdm.KeyEnableLocalMetrics, mdm.KeyLocalMetricsAddress}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.want, MDMConflicts(tc.input, policy))
})
}
}
func boolPtr(b bool) *bool { return &b }
@@ -14,6 +14,7 @@ import (
"github.com/netbirdio/netbird/client/iface"
"github.com/netbirdio/netbird/client/internal/routemanager/dynamic"
"github.com/netbirdio/netbird/client/mdm"
"github.com/netbirdio/netbird/util"
)
@@ -271,6 +272,83 @@ func TestUpdateConfigServerSSHAllowedNotSet(t *testing.T) {
}
}
func TestUpdateConfigRemoteJobsAllowed(t *testing.T) {
// Unlike SSH (which defaults on for legacy configs), remote jobs are an
// explicit opt-in: a pre-existing config with no value materializes to off.
t.Run("legacy config defaults off", func(t *testing.T) {
configPath := filepath.Join(t.TempDir(), "config.json")
require.NoError(t, os.WriteFile(configPath, []byte("{}"), 0600))
config, err := UpdateConfig(ConfigInput{ConfigPath: configPath})
require.NoError(t, err)
require.NotNil(t, config.RemoteJobsAllowed, "RemoteJobsAllowed should be materialized")
assert.False(t, *config.RemoteJobsAllowed, "remote jobs must default off")
})
for _, tt := range []struct {
name string
input *bool
want bool
}{
{"enable", util.True(), true},
{"disable", util.False(), false},
} {
t.Run(tt.name, func(t *testing.T) {
configPath := filepath.Join(t.TempDir(), "config.json")
require.NoError(t, os.WriteFile(configPath, []byte("{}"), 0600))
config, err := UpdateConfig(ConfigInput{ConfigPath: configPath, RemoteJobsAllowed: tt.input})
require.NoError(t, err)
require.NotNil(t, config.RemoteJobsAllowed)
assert.Equal(t, tt.want, *config.RemoteJobsAllowed)
})
}
}
func TestApplyMDMPolicyRemoteJobs(t *testing.T) {
t.Run("enables remote jobs and sets the upload URL override", func(t *testing.T) {
cfg := &Config{}
cfg.applyMDMPolicy(mdm.NewPolicy(map[string]any{
mdm.KeyRemoteJobsAllowed: true,
mdm.KeyBundleUploadURL: "https://upload.example.com",
}))
require.NotNil(t, cfg.RemoteJobsAllowed)
assert.True(t, *cfg.RemoteJobsAllowed, "MDM allowRemoteJobs must enable the flag")
assert.Equal(t, "https://upload.example.com", cfg.DebugBundleUploadURL, "MDM upload URL override must be applied")
})
t.Run("a non-https upload URL is rejected", func(t *testing.T) {
cfg := &Config{}
cfg.applyMDMPolicy(mdm.NewPolicy(map[string]any{
mdm.KeyBundleUploadURL: "http://insecure.example.com",
}))
assert.Empty(t, cfg.DebugBundleUploadURL, "a non-https upload URL must be skipped")
})
t.Run("dropping the key clears a previously-applied override", func(t *testing.T) {
cfg := &Config{DebugBundleUploadURL: "https://old.example.com"}
// A replacement policy that no longer carries the key must not leave
// the old upload target directing bundles.
cfg.applyMDMPolicy(mdm.NewPolicy(map[string]any{mdm.KeyRemoteJobsAllowed: true}))
assert.Empty(t, cfg.DebugBundleUploadURL, "the stale upload URL override must be cleared")
})
t.Run("an empty replacement policy clears a previously-applied override", func(t *testing.T) {
cfg := &Config{DebugBundleUploadURL: "https://old.example.com"}
// A policy that becomes empty entirely hits the IsEmpty early return;
// the override must still be cleared rather than surviving on the
// reused Config instance.
cfg.applyMDMPolicy(mdm.NewPolicy(map[string]any{}))
assert.Empty(t, cfg.DebugBundleUploadURL, "the stale upload URL override must be cleared when the policy empties")
})
t.Run("an invalid upload URL clears a previously-applied override (fail closed)", func(t *testing.T) {
cfg := &Config{DebugBundleUploadURL: "https://old.example.com"}
cfg.applyMDMPolicy(mdm.NewPolicy(map[string]any{mdm.KeyBundleUploadURL: "not-a-url"}))
assert.Empty(t, cfg.DebugBundleUploadURL, "an invalid override must fail closed, not keep the stale target")
})
}
func TestUpdateOldManagementURL(t *testing.T) {
origProber := newMgmProber
newMgmProber = func(_ context.Context, _ string, _ wgtypes.Key, _ bool) (mgmProber, error) {
@@ -0,0 +1,121 @@
package profilemanager
import (
"fmt"
"os"
"os/user"
"path/filepath"
"runtime"
"strconv"
log "github.com/sirupsen/logrus"
)
const envSudoUser = "SUDO_USER"
var (
currentUser = user.Current
getegid = os.Getegid
geteuid = os.Geteuid
lookupUser = user.Lookup
)
// InvokingUser returns the user a CLI invocation acts for. Under sudo that is
// the user who ran sudo, not root: privileged flags force commands through
// sudo, and resolving profiles as root would silently switch the daemon to
// root's (default) profile instead of the invoking user's. An unmapped positive
// process UID uses its numeric kernel identity; root, sudo lookup failures, and
// unavailable platform identities still fail closed. Privilege decisions stay
// on the kernel credentials of the daemon connection, which SUDO_USER (a plain
// environment variable) can never influence; a forged value only selects a
// profile root could select anyway.
func InvokingUser() (*user.User, error) {
if u, ok := sudoInvokingUser(); ok {
return u, nil
}
// Fail closed instead of falling through to root: every caller feeds this
// username into profile-path resolution, so a lookup failure would resolve
// (and create) a root-owned profile namespace and switch the daemon onto it
// behind the invoking user's back.
if sudoActive() {
return nil, fmt.Errorf("resolve sudo invoking user %q: refusing to fall back to root", os.Getenv(envSudoUser))
}
u, err := currentUser()
if err == nil {
return u, nil
}
uid := geteuid()
if uid <= 0 {
return nil, err
}
log.Debugf("current user lookup for UID %d: %v; using numeric UID", uid, err)
uidString := strconv.Itoa(uid)
return &user.User{
Username: uidString,
Uid: uidString,
Gid: strconv.Itoa(getegid()),
}, nil
}
// IsPlainRoot reports that the process runs as root with no usable sudo
// context: there is no invoking user to act for, so per-user resolution falls
// back to root's own (empty) state. Callers use it to refuse ambiguous
// operations instead of silently acting on the wrong profile.
func IsPlainRoot() bool {
if geteuid() != 0 {
return false
}
_, ok := sudoInvokingUser()
return !ok
}
// MirrorIsAuthoritative reports whether the invoking user's local
// active-profile mirror can be trusted as the profile selector. It cannot under
// sudo (writes to it are skipped, so it goes stale) or as plain root (there is
// no invoking user, so it falls back to root's own default). Callers use it to
// decide whether to read the profile from the mirror or from the daemon.
func MirrorIsAuthoritative() bool {
return !sudoActive() && !IsPlainRoot()
}
// sudoInvokingUser resolves SUDO_USER when the process runs as root under
// sudo. Returns false whenever the sudo context is absent or unusable, in
// which case callers fall back to the process user.
func sudoInvokingUser() (*user.User, bool) {
if !sudoActive() {
return nil, false
}
name := os.Getenv(envSudoUser)
u, err := lookupUser(name)
if err != nil {
log.Warnf("sudo invoking user %q lookup: %v", name, err)
return nil, false
}
return u, true
}
// sudoActive reports a sudo context from the environment alone: write-skip
// decisions key off it so a transient user lookup failure can never flip a
// run from read-only to writing root-owned files into the user's directory.
func sudoActive() bool {
if geteuid() != 0 {
return false
}
name := os.Getenv(envSudoUser)
return name != "" && name != "root"
}
// userBaseConfigDir mirrors os.UserConfigDir for a user other than the process
// owner. Environment overrides (XDG_CONFIG_HOME) cannot be honoured here: under
// sudo the environment is root's, not the invoking user's.
func userBaseConfigDir(u *user.User) (string, error) {
if u.HomeDir == "" {
return "", fmt.Errorf("user %s has no home directory", u.Username)
}
if runtime.GOOS == "darwin" {
return filepath.Join(u.HomeDir, "Library", "Application Support"), nil
}
return filepath.Join(u.HomeDir, ".config"), nil
}
@@ -0,0 +1,298 @@
package profilemanager
import (
"errors"
"fmt"
"io/fs"
"os"
"os/user"
"path/filepath"
"runtime"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestInvokingUserFallsBackToProcessUser(t *testing.T) {
t.Setenv(envSudoUser, "")
got, err := InvokingUser()
require.NoError(t, err)
current, err := user.Current()
require.NoError(t, err)
assert.Equal(t, current.Username, got.Username, "invoking user should match the process user without sudo")
}
func TestInvokingUserFailsClosedWithoutPositiveUID(t *testing.T) {
for _, uid := range []int{0, -1} {
t.Run(fmt.Sprintf("UID%d", uid), func(t *testing.T) {
t.Setenv(envSudoUser, "")
lookupErr := errors.New("current user unavailable")
fakeUnmappedUser(t, uid, 0, lookupErr)
got, err := InvokingUser()
require.ErrorIs(t, err, lookupErr)
assert.Nil(t, got, "root or unavailable UID must not become a synthetic identity")
})
}
}
func TestProfileFilePathUsesNumericIdentityForUnmappedNonRoot(t *testing.T) {
t.Setenv(envSudoUser, "")
fakeUnmappedUser(t, 1001230000, 0, errors.New("user: unknown userid 1001230000"))
profilesRoot := t.TempDir()
origDir := DefaultConfigPathDir
origOverride := ConfigDirOverride
DefaultConfigPathDir = profilesRoot
ConfigDirOverride = ""
t.Cleanup(func() {
DefaultConfigPathDir = origDir
ConfigDirOverride = origOverride
})
profileID := ID("0123456789abcdef0123456789abcdef")
got, err := (&Profile{ID: profileID}).FilePath()
require.NoError(t, err)
assert.Equal(t,
filepath.Join(profilesRoot, "1001230000", profileID.String()+".json"),
got,
"profile path should use the numeric UID namespace",
)
entries, err := os.ReadDir(profilesRoot)
require.NoError(t, err)
require.Len(t, entries, 1, "only the numeric UID directory should be created")
assert.Equal(t, "1001230000", entries[0].Name(), "profile namespace should be numeric")
assert.True(t, entries[0].IsDir(), "profile namespace should be a directory")
}
func TestSudoInvokingUserInactiveWithoutSudoContext(t *testing.T) {
t.Setenv(envSudoUser, "")
_, ok := sudoInvokingUser()
assert.False(t, ok)
}
func TestSudoInvokingUserIgnoresRoot(t *testing.T) {
t.Setenv(envSudoUser, "root")
origEuid := geteuid
geteuid = func() int { return 0 }
t.Cleanup(func() { geteuid = origEuid })
_, ok := sudoInvokingUser()
assert.False(t, ok, "sudo from a root shell must not redirect anything")
assert.False(t, sudoActive())
assert.True(t, IsPlainRoot())
}
func TestSudoInvokingUserResolvesInvokingUser(t *testing.T) {
fakeSudo(t, filepath.Join("/home", "misha"))
u, ok := sudoInvokingUser()
require.True(t, ok)
assert.Equal(t, "misha", u.Username)
got, err := InvokingUser()
require.NoError(t, err)
assert.Equal(t, "misha", got.Username)
assert.False(t, IsPlainRoot())
}
func TestInvokingUserFailsClosedWhenSudoLookupFails(t *testing.T) {
fakeSudo(t, filepath.Join("/home", "misha"))
lookupUser = func(string) (*user.User, error) { return nil, errors.New("nss unavailable") }
origCurrentUser := currentUser
currentUser = func() (*user.User, error) {
t.Fatal("currentUser must not be called after a sudo lookup failure")
return nil, errors.New("currentUser called unexpectedly")
}
t.Cleanup(func() { currentUser = origCurrentUser })
got, err := InvokingUser()
require.Error(t, err)
assert.Nil(t, got, "must not resolve to the root process user")
}
func TestProfileFilePathFailsClosedWhenSudoLookupFails(t *testing.T) {
profilesRoot := t.TempDir()
fakeSudo(t, filepath.Join("/home", "misha"))
lookupUser = func(string) (*user.User, error) { return nil, errors.New("nss unavailable") }
origDir := DefaultConfigPathDir
DefaultConfigPathDir = profilesRoot
t.Cleanup(func() { DefaultConfigPathDir = origDir })
p := &Profile{ID: "0123456789abcdef0123456789abcdef"}
_, err := p.FilePath()
require.Error(t, err)
assertNoEntries(t, profilesRoot)
}
func TestSudoActiveSurvivesLookupFailure(t *testing.T) {
fakeSudo(t, filepath.Join("/home", "misha"))
lookupUser = func(string) (*user.User, error) { return nil, errors.New("nss unavailable") }
_, ok := sudoInvokingUser()
assert.False(t, ok)
assert.True(t, sudoActive())
assert.True(t, IsPlainRoot())
}
func TestGetConfigDirUnderSudoIsReadOnly(t *testing.T) {
home := t.TempDir()
fakeSudo(t, home)
base, err := baseConfigDir()
require.NoError(t, err)
if runtime.GOOS == "darwin" {
assert.Equal(t, filepath.Join(home, "Library", "Application Support"), base)
} else {
assert.Equal(t, filepath.Join(home, ".config"), base)
}
dir, err := getConfigDir()
require.NoError(t, err)
assert.Equal(t, filepath.Join(base, "netbird"), dir)
assert.NoDirExists(t, dir)
}
func TestBaseConfigDirFailsClosedWhenSudoLookupFails(t *testing.T) {
fakeSudo(t, filepath.Join("/home", "misha"))
lookupUser = func(string) (*user.User, error) { return nil, errors.New("nss unavailable") }
_, err := baseConfigDir()
require.Error(t, err)
_, err = getConfigDir()
require.Error(t, err)
}
func TestSwitchProfileSkipsStateWriteUnderSudo(t *testing.T) {
home := t.TempDir()
fakeSudo(t, home)
pm := NewProfileManager()
require.NoError(t, pm.SwitchProfile(defaultProfileName))
assertNoEntries(t, home)
}
func TestSetProfileStateSkipsWriteUnderSudo(t *testing.T) {
home := t.TempDir()
fakeSudo(t, home)
pm := NewProfileManager()
require.NoError(t, pm.SetProfileState(defaultProfileName, &ProfileState{Email: "misha@example.com"}))
assertNoEntries(t, home)
}
func TestRemoveProfileStateSkipsRemoveUnderSudo(t *testing.T) {
home := t.TempDir()
stateDir := filepath.Join(home, ".config", "netbird")
if runtime.GOOS == "darwin" {
stateDir = filepath.Join(home, "Library", "Application Support", "netbird")
}
require.NoError(t, os.MkdirAll(stateDir, 0o700))
stateFile := filepath.Join(stateDir, "default.state.json")
require.NoError(t, os.WriteFile(stateFile, []byte(`{"email":"misha@example.com"}`), 0o600))
fakeSudo(t, home)
pm := NewProfileManager()
require.NoError(t, pm.RemoveProfileState("default"))
assert.FileExists(t, stateFile)
}
func TestUserBaseConfigDir(t *testing.T) {
u := &user.User{Username: "misha", HomeDir: filepath.Join("/home", "misha")}
dir, err := userBaseConfigDir(u)
require.NoError(t, err)
if runtime.GOOS == "darwin" {
assert.Equal(t, filepath.Join(u.HomeDir, "Library", "Application Support"), dir)
} else {
assert.Equal(t, filepath.Join(u.HomeDir, ".config"), dir)
}
_, err = userBaseConfigDir(&user.User{Username: "nohome"})
require.Error(t, err)
}
func TestIsPlainRoot(t *testing.T) {
t.Setenv(envSudoUser, "")
origEuid := geteuid
t.Cleanup(func() { geteuid = origEuid })
geteuid = func() int { return 1000 }
assert.False(t, IsPlainRoot())
geteuid = func() int { return 0 }
assert.True(t, IsPlainRoot())
}
func TestMirrorIsAuthoritative(t *testing.T) {
t.Setenv(envSudoUser, "")
origEuid := geteuid
t.Cleanup(func() { geteuid = origEuid })
geteuid = func() int { return 1000 }
assert.True(t, MirrorIsAuthoritative(), "a normal user's own mirror is authoritative")
geteuid = func() int { return 0 }
assert.False(t, MirrorIsAuthoritative(), "plain root has no authoritative mirror")
}
func TestMirrorIsAuthoritativeFalseUnderSudo(t *testing.T) {
fakeSudo(t, filepath.Join("/home", "misha"))
assert.False(t, MirrorIsAuthoritative(), "the sudo mirror is frozen, so it is not authoritative")
}
func fakeSudo(t *testing.T, home string) {
t.Helper()
t.Setenv(envSudoUser, "misha")
origEuid := geteuid
origLookup := lookupUser
origOverride := ConfigDirOverride
geteuid = func() int { return 0 }
lookupUser = func(name string) (*user.User, error) {
return &user.User{Username: name, Uid: "1234", Gid: "1234", HomeDir: home}, nil
}
ConfigDirOverride = ""
t.Cleanup(func() {
geteuid = origEuid
lookupUser = origLookup
ConfigDirOverride = origOverride
})
}
func fakeUnmappedUser(t *testing.T, uid, gid int, lookupErr error) {
t.Helper()
origCurrentUser := currentUser
origEuid := geteuid
origEgid := getegid
currentUser = func() (*user.User, error) { return nil, lookupErr }
geteuid = func() int { return uid }
getegid = func() int { return gid }
t.Cleanup(func() {
currentUser = origCurrentUser
geteuid = origEuid
getegid = origEgid
})
}
func assertNoEntries(t *testing.T, root string) {
t.Helper()
err := filepath.WalkDir(root, func(path string, _ fs.DirEntry, err error) error {
if err != nil {
return err
}
if path != root {
t.Errorf("unexpected entry created under %s: %s", root, path)
}
return nil
})
require.NoError(t, err)
}
@@ -3,7 +3,6 @@ package profilemanager
import (
"fmt"
"os"
"os/user"
"path/filepath"
"strings"
"sync"
@@ -54,7 +53,7 @@ func (p *Profile) FilePath() (string, error) {
return "", fmt.Errorf("invalid profile ID: %q", id)
}
username, err := user.Current()
username, err := InvokingUser()
if err != nil {
return "", fmt.Errorf("failed to get current user: %w", err)
}
@@ -130,7 +129,7 @@ func (pm *ProfileManager) getActiveProfileState() ID {
if err != nil {
if !os.IsNotExist(err) {
log.Warnf("failed to read active profile state: %v", err)
} else {
} else if !sudoActive() {
if err := pm.setActiveProfileState(defaultProfileName); err != nil {
log.Warnf("failed to set default profile state: %v", err)
}
@@ -148,6 +147,13 @@ func (pm *ProfileManager) getActiveProfileState() ID {
}
func (pm *ProfileManager) setActiveProfileState(id ID) error {
// The invoking user's state is read-only under sudo — a root-owned file in
// the user's directory would break their own runs. The daemon still records
// the switch on its side; only the user-local bookkeeping is skipped.
if sudoActive() {
log.Infof("running under sudo: not persisting active profile %q for user %s", id, os.Getenv(envSudoUser))
return nil
}
configDir, err := getConfigDir()
if err != nil {
+16
View File
@@ -7,6 +7,8 @@ import (
"os"
"path/filepath"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/util"
)
@@ -63,6 +65,15 @@ func (pm *ProfileManager) SetProfileState(id ID, state *ProfileState) error {
return fmt.Errorf("invalid profile ID: %q", id)
}
// The invoking user's state is read-only under sudo. The file only carries
// the account email for the login hint and display, so skipping the write
// costs at most one extra account prompt later — a root-owned file in the
// user's directory would cost every later update instead.
if sudoActive() {
log.Debugf("running under sudo: not persisting profile state for user %s", os.Getenv(envSudoUser))
return nil
}
stateFile := filepath.Join(configDir, id.String()+".state.json")
if err := util.WriteJsonWithRestrictedPermission(context.Background(), stateFile, state); err != nil {
return fmt.Errorf("write profile state: %w", err)
@@ -92,6 +103,11 @@ func (pm *ProfileManager) SetActiveProfileState(state *ProfileState) error {
// equivalent to clearing it; the next SSO login recreates it. A missing file
// is not an error.
func (pm *ProfileManager) RemoveProfileState(profileName string) error {
if sudoActive() {
log.Debugf("running under sudo: not removing profile state for user %s", os.Getenv(envSudoUser))
return nil
}
configDir, err := getConfigDir()
if err != nil {
return fmt.Errorf("get config directory: %w", err)
+2 -10
View File
@@ -201,11 +201,7 @@ func (p *StunTurnProbe) probeSTUN(ctx context.Context, uri *stun.URI) (addr stri
}
}()
net, err := stdnet.NewNet(ctx, nil)
if err != nil {
probeErr = fmt.Errorf("new net: %w", err)
return
}
net := stdnet.NewNet(ctx, nil)
client, err := stun.DialURI(uri, &stun.DialConfig{
Net: net,
@@ -290,11 +286,7 @@ func (p *StunTurnProbe) probeTURN(ctx context.Context, uri *stun.URI) (addr stri
}
}()
net, err := stdnet.NewNet(ctx, nil)
if err != nil {
probeErr = fmt.Errorf("new net: %w", err)
return
}
net := stdnet.NewNet(ctx, nil)
cfg := &turn.ClientConfig{
STUNServerAddr: turnServerAddr,
TURNServerAddr: turnServerAddr,
@@ -294,6 +294,7 @@ func (w *Watcher) addAllowedIPs(route *route.Route) error {
return fmt.Errorf("add allowed IPs for peer %s: %w", route.Peer, err)
}
w.statusRecorder.AddActiveRoutePeer(route.GetHAUniqueID(), route.Peer)
if err := w.statusRecorder.AddPeerStateRoute(route.Peer, w.handler.String(), route.GetResourceID()); err != nil {
log.Warnf("Failed to update peer state: %v", err)
}
@@ -303,6 +304,7 @@ func (w *Watcher) addAllowedIPs(route *route.Route) error {
}
func (w *Watcher) removeAllowedIPs(route *route.Route, rsn reason) error {
w.statusRecorder.RemoveActiveRoutePeer(route.GetHAUniqueID())
if err := w.statusRecorder.RemovePeerStateRoute(route.Peer, w.handler.String()); err != nil {
log.Warnf("Failed to update peer state: %v", err)
}
+41 -18
View File
@@ -8,6 +8,7 @@ import (
"net/netip"
"net/url"
"runtime"
"slices"
"sort"
"strings"
"sync"
@@ -472,27 +473,13 @@ func (m *DefaultManager) CurrentRouteRange() []string {
m.mux.Lock()
defer m.mux.Unlock()
if m.disableClientRoutes {
return nil
}
filtered := m.routeSelector.FilterSelectedExitNodes(m.clientRoutes)
var nets []string
for _, routes := range filtered {
for _, r := range routes {
if r.IsDynamic() {
continue
}
nets = append(nets, r.NetString())
}
}
if m.fakeIPManager != nil {
nets = append(nets, m.fakeIPManager.GetFakeIPBlock().String(), m.fakeIPManager.GetFakeIPv6Block().String())
nets := m.overlayNetworks()
if !m.disableClientRoutes {
nets = append(nets, m.clientRouteRange()...)
}
sort.Strings(nets)
return nets
return slices.Compact(nets)
}
// GetRouteSelector returns the route selector
@@ -856,6 +843,42 @@ func (m *DefaultManager) logExitNodeUpdate(info exitNodeInfo, preferred route.Ne
len(info.allIDs), preferred, len(info.userSelected), len(info.userDeselected), len(info.selectedByManagement))
}
// overlayNetworks returns the v4 and v6 overlay networks of the WireGuard interface, each only when it is set.
func (m *DefaultManager) overlayNetworks() []string {
if m.wgInterface == nil {
return nil
}
addr := m.wgInterface.Address()
var nets []string
if addr.Network.IsValid() {
nets = append(nets, addr.Network.String())
}
if addr.IPv6Net.IsValid() {
nets = append(nets, addr.IPv6Net.String())
}
return nets
}
// clientRouteRange returns the static client route networks of the selected exit nodes together with the fake IP blocks.
func (m *DefaultManager) clientRouteRange() []string {
filtered := m.routeSelector.FilterSelectedExitNodes(m.clientRoutes)
var nets []string
for _, routes := range filtered {
for _, r := range routes {
if r.IsDynamic() {
continue
}
nets = append(nets, r.NetString())
}
}
if m.fakeIPManager != nil {
nets = append(nets, m.fakeIPManager.GetFakeIPBlock().String(), m.fakeIPManager.GetFakeIPv6Block().String())
}
return nets
}
// minNetID returns the lexicographically smallest NetID, for a deterministic
// default pick that stays stable across restarts.
func minNetID(ids []route.NetID) route.NetID {
+2 -4
View File
@@ -8,6 +8,7 @@ import (
"net/netip"
"testing"
"github.com/netbirdio/netbird/client/internal/profilemanager"
"github.com/netbirdio/netbird/client/internal/stdnet"
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
@@ -406,10 +407,7 @@ func TestManagerUpdateRoutes(t *testing.T) {
for n, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
peerPrivateKey, _ := wgtypes.GeneratePrivateKey()
newNet, err := stdnet.NewNet(context.Background(), nil)
if err != nil {
t.Fatal(err)
}
newNet := stdnet.NewNet(context.Background(), profilemanager.DefaultInterfaceBlacklist)
opts := iface.WGIFaceOpts{
IFaceName: fmt.Sprintf("utun43%d", n),
Address: wgaddr.MustParseWGAddress("100.65.65.2/24"),
@@ -4,8 +4,6 @@ package notifier
import (
"net/netip"
"slices"
"sort"
"sync"
"github.com/netbirdio/netbird/client/internal/listener"
@@ -75,19 +73,3 @@ func (n *Notifier) notifyLocked() {
func (n *Notifier) Close() {
// unused
}
func routesToStrings(routes []*route.Route) []string {
nets := make([]string, 0, len(routes))
for _, r := range routes {
nets = append(nets, r.NetString())
}
return nets
}
func hasRouteDiff(a []*route.Route, b []*route.Route) bool {
as := routesToStrings(a)
bs := routesToStrings(b)
sort.Strings(as)
sort.Strings(bs)
return !slices.Equal(as, bs)
}
@@ -0,0 +1,27 @@
package notifier
import (
"slices"
"sort"
"github.com/netbirdio/netbird/route"
)
// routePrefixes returns the distinct prefixes a route set covers, sorted.
// Duplicates are dropped deliberately: an HA group hands us one route per
// peer serving the same prefix, and the platform is given the prefix, not the
// candidates. Counting them would report a change every time a peer joins or
// leaves a group, and on Android each report renews the TUN.
func routePrefixes(routes []*route.Route) []string {
nets := make([]string, 0, len(routes))
for _, r := range routes {
nets = append(nets, r.NetString())
}
sort.Strings(nets)
return slices.Compact(nets)
}
// hasRouteDiff reports whether the prefixes the two route sets cover differ.
func hasRouteDiff(a []*route.Route, b []*route.Route) bool {
return !slices.Equal(routePrefixes(a), routePrefixes(b))
}
@@ -0,0 +1,88 @@
package notifier
import (
"net/netip"
"testing"
"github.com/stretchr/testify/assert"
"github.com/netbirdio/netbird/route"
)
func routeFor(id route.ID, prefix string) *route.Route {
return &route.Route{
ID: id,
NetID: "net",
Network: netip.MustParsePrefix(prefix),
}
}
// TestHasRouteDiff_IgnoresHACandidateCount is the reason the comparison
// deduplicates. Every notification renews the TUN, and a renewed TUN
// invalidates the sockets the embedded servers are listening on, so a peer
// joining or leaving an HA group must not count as a route change when the
// prefixes the TUN carries are identical.
func TestHasRouteDiff_IgnoresHACandidateCount(t *testing.T) {
onePeer := []*route.Route{routeFor("a", "10.0.0.0/24")}
twoPeers := []*route.Route{
routeFor("a", "10.0.0.0/24"),
routeFor("b", "10.0.0.0/24"),
}
assert.False(t, hasRouteDiff(onePeer, twoPeers),
"a second peer serving the same prefix is not a route change")
assert.False(t, hasRouteDiff(twoPeers, onePeer),
"losing one of two peers serving the same prefix is not a route change")
}
func TestHasRouteDiff_ReportsRealChanges(t *testing.T) {
tests := []struct {
name string
a []*route.Route
b []*route.Route
want bool
}{
{
name: "added prefix",
a: []*route.Route{routeFor("a", "10.0.0.0/24")},
b: []*route.Route{routeFor("a", "10.0.0.0/24"), routeFor("b", "10.0.1.0/24")},
want: true,
},
{
name: "removed prefix",
a: []*route.Route{routeFor("a", "10.0.0.0/24"), routeFor("b", "10.0.1.0/24")},
b: []*route.Route{routeFor("a", "10.0.0.0/24")},
want: true,
},
{
name: "replaced prefix",
a: []*route.Route{routeFor("a", "10.0.0.0/24")},
b: []*route.Route{routeFor("a", "10.0.1.0/24")},
want: true,
},
{
name: "same prefix, different order",
a: []*route.Route{routeFor("a", "10.0.1.0/24"), routeFor("b", "10.0.0.0/24")},
b: []*route.Route{routeFor("b", "10.0.0.0/24"), routeFor("a", "10.0.1.0/24")},
want: false,
},
{
name: "all routes gone",
a: []*route.Route{routeFor("a", "10.0.0.0/24")},
b: nil,
want: true,
},
{
name: "both empty",
a: nil,
b: nil,
want: false,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.want, hasRouteDiff(tc.a, tc.b),
"route diff for %s", tc.name)
})
}
}
@@ -17,11 +17,12 @@ import (
"github.com/netbirdio/netbird/client/internal/routemanager/refcounter"
)
// reconcileWGMock is a minimal iface.WGIface that only records AddAllowedIP calls; every other
// method is an inert stub because ReconcilePeerAllowedIPs exercises none of them.
// reconcileWGMock is a minimal iface.WGIface that records AddAllowedIP calls and reports the
// configured address; every other method is an inert stub because the tests exercise none of them.
type reconcileWGMock struct {
mu sync.Mutex
adds map[string][]netip.Prefix
addr wgaddr.Address
}
func (m *reconcileWGMock) AddAllowedIP(peerKey string, allowedIP netip.Prefix) error {
@@ -42,7 +43,7 @@ func (m *reconcileWGMock) added(peerKey string) []netip.Prefix {
func (m *reconcileWGMock) RemoveAllowedIP(string, netip.Prefix) error { return nil }
func (m *reconcileWGMock) Name() string { return "utun-test" }
func (m *reconcileWGMock) Address() wgaddr.Address { return wgaddr.Address{} }
func (m *reconcileWGMock) Address() wgaddr.Address { return m.addr }
func (m *reconcileWGMock) ToInterface() *net.Interface { return nil }
func (m *reconcileWGMock) IsUserspaceBind() bool { return false }
func (m *reconcileWGMock) GetFilter() device.PacketFilter { return nil }
@@ -0,0 +1,95 @@
//go:build !windows
package routemanager
import (
"net/netip"
"testing"
"github.com/stretchr/testify/assert"
"github.com/netbirdio/netbird/client/iface/wgaddr"
"github.com/netbirdio/netbird/client/internal/routeselector"
"github.com/netbirdio/netbird/route"
)
func TestCurrentRouteRange_OverlayNetworkWithClientRoutesDisabled(t *testing.T) {
m := &DefaultManager{
wgInterface: &reconcileWGMock{addr: wgaddr.MustParseWGAddress("100.91.96.107/16")},
disableClientRoutes: true,
}
assert.Equal(t, []string{"100.91.0.0/16"}, m.CurrentRouteRange(), "overlay network must be routed even when client routes are disabled")
}
func TestCurrentRouteRange_OverlayNetworksAndClientRoutes(t *testing.T) {
addr := wgaddr.MustParseWGAddress("100.91.96.107/16")
addr.IPv6 = netip.MustParseAddr("fd00:1234::1")
addr.IPv6Net = netip.MustParsePrefix("fd00:1234::/64")
static := &route.Route{ID: "static", NetID: "lan", Network: netip.MustParsePrefix("192.168.50.0/24"), NetworkType: route.IPv4Network}
dynamic := &route.Route{ID: "dynamic", NetID: "dyn", NetworkType: route.DomainNetwork}
m := &DefaultManager{
wgInterface: &reconcileWGMock{addr: addr},
routeSelector: routeselector.NewRouteSelector(),
clientRoutes: route.HAMap{
static.GetHAUniqueID(): {static},
dynamic.GetHAUniqueID(): {dynamic},
},
}
assert.Equal(t, []string{"100.91.0.0/16", "192.168.50.0/24", "fd00:1234::/64"}, m.CurrentRouteRange(), "overlay networks and static client routes must be listed, dynamic routes skipped")
}
func TestCurrentRouteRange_NoInterfaceAddress(t *testing.T) {
m := &DefaultManager{
wgInterface: &reconcileWGMock{},
disableClientRoutes: true,
}
assert.Empty(t, m.CurrentRouteRange(), "an unset interface address must not produce a route entry")
}
func TestCurrentRouteRange_IPv6WithoutIPv4Network(t *testing.T) {
addr := wgaddr.Address{
IPv6: netip.MustParseAddr("fd00:1234::1"),
IPv6Net: netip.MustParsePrefix("fd00:1234::/64"),
}
m := &DefaultManager{
wgInterface: &reconcileWGMock{addr: addr},
disableClientRoutes: true,
}
assert.Equal(t, []string{"fd00:1234::/64"}, m.CurrentRouteRange(), "a v6 overlay network must not depend on a v4 network being set")
}
func TestCurrentRouteRange_IPv6AddressWithoutNetwork(t *testing.T) {
addr := wgaddr.MustParseWGAddress("100.91.96.107/16")
addr.IPv6 = netip.MustParseAddr("fd00:1234::1")
m := &DefaultManager{
wgInterface: &reconcileWGMock{addr: addr},
disableClientRoutes: true,
}
assert.Equal(t, []string{"100.91.0.0/16"}, m.CurrentRouteRange(), "a v6 address without a network must not produce a route entry")
}
func TestCurrentRouteRange_DeduplicatesPrefixes(t *testing.T) {
// Two HA peers serve the same prefix, and a client route announces the overlay network itself.
haPeerA := &route.Route{ID: "ha-a", NetID: "lan", Peer: "peer-a", Network: netip.MustParsePrefix("192.168.50.0/24"), NetworkType: route.IPv4Network}
haPeerB := &route.Route{ID: "ha-b", NetID: "lan", Peer: "peer-b", Network: netip.MustParsePrefix("192.168.50.0/24"), NetworkType: route.IPv4Network}
overlay := &route.Route{ID: "overlay", NetID: "overlay", Network: netip.MustParsePrefix("100.91.0.0/16"), NetworkType: route.IPv4Network}
m := &DefaultManager{
wgInterface: &reconcileWGMock{addr: wgaddr.MustParseWGAddress("100.91.96.107/16")},
routeSelector: routeselector.NewRouteSelector(),
clientRoutes: route.HAMap{
haPeerA.GetHAUniqueID(): {haPeerA, haPeerB},
overlay.GetHAUniqueID(): {overlay},
},
}
assert.Equal(t, []string{"100.91.0.0/16", "192.168.50.0/24"}, m.CurrentRouteRange(), "every prefix must be listed once regardless of how many routes carry it")
}
+16 -9
View File
@@ -17,23 +17,30 @@ import (
// are mutually exclusive: if the selection activates an exit node, every other
// available exit node is deselected so two can't be active at once. With
// appendRoute=false the previous selection is replaced instead of extended.
// A partial failure (e.g. an unknown ID mixed with valid ones) still applies
// the valid IDs to the routing table; the unknown ones are reported in the
// returned error.
func (m *DefaultManager) SelectRoutes(ids []route.NetID, appendRoute bool) error {
if err := m.selectRoutes(ids, appendRoute); err != nil {
return err
}
err := m.selectRoutes(ids, appendRoute)
// Apply regardless of err: selectRoutes already selects the valid part of a
// partial request, and skipping this on error would leave those routes
// selected in the selector but never installed in the routing table.
m.TriggerSelection(m.GetClientRoutes())
return nil
return err
}
// DeselectRoutes removes the routes with the given network IDs from the
// selection and applies the change. V4/v6 exit-node pairs are expanded
// automatically.
// automatically. A partial failure (e.g. an unknown ID mixed with valid ones)
// still applies the valid IDs to the routing table; the unknown ones are
// reported in the returned error.
func (m *DefaultManager) DeselectRoutes(ids []route.NetID) error {
if err := m.deselectRoutes(ids); err != nil {
return err
}
err := m.deselectRoutes(ids)
// Apply regardless of err: deselectRoutes already deselects the valid part
// of a partial request, and skipping this on error would leave those routes
// installed in the routing table despite being marked deselected.
m.TriggerSelection(m.GetClientRoutes())
return nil
return err
}
func (m *DefaultManager) deselectRoutes(ids []route.NetID) error {
@@ -1,12 +1,17 @@
package routemanager
import (
"context"
"net/netip"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/exp/maps"
"github.com/netbirdio/netbird/client/internal/peer"
"github.com/netbirdio/netbird/client/internal/routemanager/client"
"github.com/netbirdio/netbird/client/internal/routemanager/notifier"
"github.com/netbirdio/netbird/client/internal/routeselector"
"github.com/netbirdio/netbird/route"
)
@@ -112,6 +117,75 @@ func TestSelectRoutes_UnknownRoute(t *testing.T) {
assert.Error(t, m.deselectRoutes([]route.NetID{"missing"}), "deselecting an unavailable route must fail")
}
// newPartialFailureTestManager exercises the real install/remove path without
// touching the system: the noop refcounter absorbs the route changes, and every
// route already has a watcher, so none is started.
func newPartialFailureTestManager() *DefaultManager {
ctx := context.Background()
m := &DefaultManager{
ctx: ctx,
clientRoutes: route.HAMap{
"lan|192.168.1.0/24": {{NetID: "lan", Network: netip.MustParsePrefix("192.168.1.0/24"), Peer: "p1"}},
"other|10.1.2.0/24": {{NetID: "other", Network: netip.MustParsePrefix("10.1.2.0/24"), Peer: "p2"}},
},
routeSelector: routeselector.NewRouteSelector(),
notifier: notifier.NewNotifier(),
statusRecorder: peer.NewRecorder("https://mgm"),
activeRoutes: make(map[route.HAUniqueID]client.RouteHandler),
clientNetworks: map[route.HAUniqueID]*client.Watcher{
"lan|192.168.1.0/24": client.NewWatcher(client.WatcherConfig{Context: ctx}),
"other|10.1.2.0/24": client.NewWatcher(client.WatcherConfig{Context: ctx}),
},
}
m.setupRefCounters(true)
return m
}
// Regression for the reported symptom: a partial failure returned before
// TriggerSelection ran, so the valid route was marked selected while never
// reaching the routing table (activeRoutes/ip route).
func TestSelectRoutes_PartialFailureStillInstallsValidRoute(t *testing.T) {
m := newPartialFailureTestManager()
err := m.SelectRoutes([]route.NetID{"missing", "lan"}, false)
assert.Error(t, err, "the unknown id must still be reported")
assert.Contains(t, m.activeRoutes, route.HAUniqueID("lan|192.168.1.0/24"), "the valid route must be installed despite the error")
assert.NotContains(t, m.activeRoutes, route.HAUniqueID("other|10.1.2.0/24"), "the deselected route must not be installed")
}
// Mirror of the case above: a partial failure must remove the valid route from
// the routing table, not just mark it deselected in the selector.
func TestDeselectRoutes_PartialFailureStillRemovesValidRoute(t *testing.T) {
m := newPartialFailureTestManager()
require.NoError(t, m.SelectRoutes([]route.NetID{"lan", "other"}, false))
require.Contains(t, m.activeRoutes, route.HAUniqueID("lan|192.168.1.0/24"))
require.Contains(t, m.activeRoutes, route.HAUniqueID("other|10.1.2.0/24"))
err := m.DeselectRoutes([]route.NetID{"missing", "other"})
assert.Error(t, err, "the unknown id must still be reported")
assert.NotContains(t, m.activeRoutes, route.HAUniqueID("other|10.1.2.0/24"), "the deselected route must be removed")
assert.Contains(t, m.activeRoutes, route.HAUniqueID("lan|192.168.1.0/24"), "the untouched route stays installed")
}
// The selection now runs on every request, including one where no ID is known
// and the selector stays untouched. Nothing may be torn down or reinstalled on
// that path.
func TestSelectRoutes_TotalFailureLeavesInstalledRoutesAlone(t *testing.T) {
m := newPartialFailureTestManager()
require.NoError(t, m.SelectRoutes([]route.NetID{"lan", "other"}, false))
installed := maps.Keys(m.activeRoutes)
err := m.SelectRoutes([]route.NetID{"missing"}, false)
assert.Error(t, err, "the unknown id must still be reported")
assert.ElementsMatch(t, installed, maps.Keys(m.activeRoutes), "a fully invalid request must not disturb the routing table")
}
func TestExitNodeSelectionHelpers(t *testing.T) {
routesMap := map[route.NetID][]*route.Route{
"exitA": {{Network: netip.MustParsePrefix("0.0.0.0/0")}},
@@ -135,6 +135,14 @@ func (r *Router) CleanUp() {
}
}
// Give back the routing reference taken in UpdateRoutes, after the routes
// are gone as above. Without this the sysctls enabling it changed (IPv6
// forwarding and the accept_ra values that keep RA handling alive next to
// it) stay applied once the client stops.
if err := r.firewall.DisableRouting(); err != nil {
log.Errorf("Failed to disable routing: %v", err)
}
r.statusRecorder.CleanLocalPeerStateRoutes()
}
@@ -0,0 +1,66 @@
package server
import (
"context"
"net/netip"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
firewall "github.com/netbirdio/netbird/client/firewall/manager"
"github.com/netbirdio/netbird/client/internal/peer"
"github.com/netbirdio/netbird/route"
)
// routingFirewall records the routing lifecycle calls the router makes. The
// embedded interface covers the methods this test never reaches.
type routingFirewall struct {
firewall.Manager
removed []firewall.RouterPair
enabled int
disabled int
}
func (f *routingFirewall) RemoveNatRule(pair firewall.RouterPair) error {
f.removed = append(f.removed, pair)
return nil
}
func (f *routingFirewall) EnableRouting() error {
f.enabled++
return nil
}
func (f *routingFirewall) DisableRouting() error {
f.disabled++
return nil
}
// TestRouterCleanUpReleasesRouting covers the shutdown path: the router holds a
// routing reference for as long as it serves routes, and CleanUp has to give it
// back. Without that the sysctls the reference enabled (IPv6 forwarding and the
// accept_ra values that keep RA handling working alongside it) stay applied
// after the client stops, leaving the host configured as a router.
func TestRouterCleanUpReleasesRouting(t *testing.T) {
fw := &routingFirewall{}
r := &Router{
ctx: context.Background(),
firewall: fw,
statusRecorder: peer.NewRecorder("https://mgm"),
routes: map[route.ID]*route.Route{
"route-1": {
ID: "route-1",
Network: netip.MustParsePrefix("192.168.55.0/24"),
NetworkType: route.IPv4Network,
Masquerade: true,
},
},
}
r.CleanUp()
require.Len(t, fw.removed, 1, "the route's NAT rule must be removed")
assert.Equal(t, 1, fw.disabled, "CleanUp must release the routing reference")
}
@@ -15,6 +15,7 @@ import (
"syscall"
"testing"
"github.com/netbirdio/netbird/client/internal/profilemanager"
"github.com/netbirdio/netbird/client/internal/stdnet"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -436,8 +437,7 @@ func createWGInterface(t *testing.T, interfaceName, ipAddressCIDR string, listen
peerPrivateKey, err := wgtypes.GeneratePrivateKey()
require.NoError(t, err)
newNet, err := stdnet.NewNet(context.Background(), nil)
require.NoError(t, err)
newNet := stdnet.NewNet(context.Background(), profilemanager.DefaultInterfaceBlacklist)
opts := iface.WGIFaceOpts{
IFaceName: interfaceName,
+19 -8
View File
@@ -32,6 +32,22 @@ func (rs *RouteSelector) SelectRoutes(routes []route.NetID, appendRoute bool, al
rs.mu.Lock()
defer rs.mu.Unlock()
// Validate before mutating: a non-append selection wipes the current selection
// first, so a request of only unavailable routes would deselect everything and
// put nothing back. An empty request means deselect all, so it still goes through.
var err *multierror.Error
available := make([]route.NetID, 0, len(routes))
for _, r := range routes {
if !slices.Contains(allRoutes, r) {
err = multierror.Append(err, fmt.Errorf("route '%s' is not available", r))
continue
}
available = append(available, r)
}
if len(available) == 0 && err != nil {
return errors.FormatErrorOrNil(err)
}
if !appendRoute || rs.deselectAll {
if rs.deselectedRoutes == nil {
rs.deselectedRoutes = map[route.NetID]struct{}{}
@@ -46,14 +62,9 @@ func (rs *RouteSelector) SelectRoutes(routes []route.NetID, appendRoute bool, al
}
}
var err *multierror.Error
for _, route := range routes {
if !slices.Contains(allRoutes, route) {
err = multierror.Append(err, fmt.Errorf("route '%s' is not available", route))
continue
}
delete(rs.deselectedRoutes, route)
rs.selectedRoutes[route] = struct{}{}
for _, r := range available {
delete(rs.deselectedRoutes, r)
rs.selectedRoutes[r] = struct{}{}
}
rs.deselectAll = false

Some files were not shown because too many files have changed in this diff Show More