sendnrw/netbird
2
0
Fork 0
mirror of https://github.com/netbirdio/netbird.git synced 2026-09-08 16:01:29 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into embedded-vnc

Browse Source
This commit is contained in:
Viktor Liu
2026-08-25 18:26:29 +02:00
parent a730672a4e c512bf25aa
commit 4b72019809
439 changed files with 35262 additions and 5730 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
client/internal/acl/manager.go
View File

@@ -116,11 +116,11 @@ func (d *DefaultManager) ApplyFiltering(networkMap *mgmProto.NetworkMap, dnsRout
// firewall state, so an identical hash means an identical resulting ruleset.
func (d *DefaultManager) firewallConfigHash(networkMap *mgmProto.NetworkMap, dnsRouteFeatureFlag bool) (uint64, error) {
return hashstructure.Hash(struct {
PeerRules []*mgmProto.FirewallRule
PeerRulesIsEmpty bool
RouteRules []*mgmProto.RouteFirewallRule
RouteRulesIsEmpty bool
DNSRouteFeatureFlag bool
PeerRules []*mgmProto.FirewallRule
PeerRulesIsEmpty bool
RouteRules []*mgmProto.RouteFirewallRule
RouteRulesIsEmpty bool
DNSRouteFeatureFlag bool
}{
PeerRules: networkMap.GetFirewallRules(),
PeerRulesIsEmpty: networkMap.GetFirewallRulesIsEmpty(),
@@ -144,13 +144,13 @@ func (d *DefaultManager) applyPeerACLs(networkMap *mgmProto.NetworkMap) {
log.Warn("this peer is connected to a NetBird Management service with an older version. Allowing all traffic from connected peers")
rules = append(rules,
&mgmProto.FirewallRule{
PeerIP: "0.0.0.0",
PeerIP: "0.0.0.0", //nolint:staticcheck
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_ALL,
},
&mgmProto.FirewallRule{
PeerIP: "0.0.0.0",
PeerIP: "0.0.0.0", //nolint:staticcheck
Direction: mgmProto.RuleDirection_OUT,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_ALL,
@@ -407,7 +407,6 @@ func (d *DefaultManager) getRuleGroupingSelector(rule *mgmProto.FirewallRule) st
return fmt.Sprintf("%v:%v:%v:%s:%v", strconv.Itoa(int(rule.Direction)), rule.Action, rule.Protocol, rule.Port, rule.PortInfo)
}
// extractRuleIP extracts the peer IP from a firewall rule.
// If sourcePrefixes is populated (new management), decode the first entry and use its address.
// Otherwise fall back to the deprecated PeerIP string field (old management).

8
client/internal/acl/manager_test.go
View File

@@ -5,9 +5,9 @@ import (
"net/netip"
"testing"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"github.com/netbirdio/netbird/client/firewall"
"github.com/netbirdio/netbird/client/iface"
@@ -87,7 +87,7 @@ func TestDefaultManager(t *testing.T) {
networkMap.FirewallRules = append(
networkMap.FirewallRules,
&mgmProto.FirewallRule{
PeerIP: "10.93.0.3",
PeerIP: "10.93.0.3", //nolint:staticcheck
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_DROP,
Protocol: mgmProto.RuleProtocol_ICMP,
@@ -556,12 +556,12 @@ func TestApplyFilteringSkipsUnchangedConfig(t *testing.T) {
func buildNetworkMap(peerRules, routeRules int) *mgmProto.NetworkMap {
nm := &mgmProto.NetworkMap{
FirewallRulesIsEmpty: peerRules == 0,
FirewallRulesIsEmpty: peerRules == 0,
RoutesFirewallRulesIsEmpty: routeRules == 0,
}
for i := range peerRules {
nm.FirewallRules = append(nm.FirewallRules, &mgmProto.FirewallRule{
PeerIP: fmt.Sprintf("10.%d.%d.%d", i>>16&0xff, i>>8&0xff, i&0xff),
PeerIP: fmt.Sprintf("10.%d.%d.%d", i>>16&0xff, i>>8&0xff, i&0xff), //nolint:staticcheck
Direction: mgmProto.RuleDirection_IN,
Action: mgmProto.RuleAction_ACCEPT,
Protocol: mgmProto.RuleProtocol_TCP,

2
client/internal/acl/mocks/iface_mapper.go
View File

@@ -7,7 +7,7 @@ package mocks
import (
reflect "reflect"
gomock "github.com/golang/mock/gomock"
gomock "go.uber.org/mock/gomock"
wgdevice "golang.zx2c4.com/wireguard/device"
"github.com/netbirdio/netbird/client/iface/device"

27
client/internal/auth/auth.go
View File

@@ -138,26 +138,37 @@ func (a *Auth) IsSSOSupported(ctx context.Context) (bool, error) {
// GetOAuthFlow returns an OAuth flow (PKCE or Device) using the existing management connection
// This avoids creating a new connection to the management server
func (a *Auth) GetOAuthFlow(ctx context.Context, forceDeviceAuth bool) (OAuthFlow, error) {
func (a *Auth) GetOAuthFlow(ctx context.Context, forceDeviceAuth bool, hint string) (OAuthFlow, error) {
var flow OAuthFlow
var err error
err = a.withRetry(ctx, func(client *mgm.GrpcClient) error {
err := a.withRetry(ctx, func(client *mgm.GrpcClient) error {
if forceDeviceAuth {
flow, err = a.getDeviceFlow(client)
return err
deviceFlow, err := a.getDeviceFlow(client)
if err != nil {
return err
}
deviceFlow.SetLoginHint(hint)
flow = deviceFlow
return nil
}
// Try PKCE flow first
flow, err = a.getPKCEFlow(client)
pkceFlow, err := a.getPKCEFlow(client)
if err != nil {
// If PKCE not supported, try Device flow
if s, ok := status.FromError(err); ok && (s.Code() == codes.NotFound || s.Code() == codes.Unimplemented) {
flow, err = a.getDeviceFlow(client)
return err
deviceFlow, err := a.getDeviceFlow(client)
if err != nil {
return err
}
deviceFlow.SetLoginHint(hint)
flow = deviceFlow
return nil
}
return err
}
pkceFlow.SetLoginHint(hint)
flow = pkceFlow
return nil
})

8
client/internal/auth/oauth.go
View File

@@ -97,9 +97,7 @@ func authenticateWithPKCEFlow(ctx context.Context, config *profilemanager.Config
return nil, fmt.Errorf("getting pkce authorization flow info failed with error: %v", err)
}
if hint != "" {
pkceFlowInfo.SetLoginHint(hint)
}
pkceFlowInfo.SetLoginHint(hint)
return pkceFlowInfo, nil
}
@@ -127,9 +125,7 @@ func authenticateWithDeviceCodeFlow(ctx context.Context, config *profilemanager.
}
}
if hint != "" {
deviceFlowInfo.SetLoginHint(hint)
}
deviceFlowInfo.SetLoginHint(hint)
return deviceFlowInfo, nil
}

63
client/internal/connect.go
View File

@@ -38,6 +38,8 @@ import (
"github.com/netbirdio/netbird/client/internal/updater"
"github.com/netbirdio/netbird/client/internal/updater/installer"
nbnet "github.com/netbirdio/netbird/client/net"
"github.com/netbirdio/netbird/client/netstate"
"github.com/netbirdio/netbird/client/netsweep"
cProto "github.com/netbirdio/netbird/client/proto"
"github.com/netbirdio/netbird/client/ssh"
sshconfig "github.com/netbirdio/netbird/client/ssh/config"
@@ -70,18 +72,42 @@ type ConnectClient struct {
updateManager *updater.Manager
persistSyncResponse bool
// netState gates every reconnection loop on OS-reported network
// availability. Nil (the default) disables gating; mobile platforms
// inject it via WithNetworkState.
netState *netstate.State
// sweeper cuts the management, signal and relay connections on network
// change; nil disables it.
sweeper *netsweep.Sweeper
}
// ConnectClientOption configures optional ConnectClient behavior.
type ConnectClientOption func(*ConnectClient)
// WithNetworkState injects the OS network availability state that gates every
// reconnection loop; without it gating is disabled.
func WithNetworkState(netState *netstate.State) ConnectClientOption {
return func(c *ConnectClient) { c.netState = netState }
}
// WithSweeper injects the network change sweeper.
func WithSweeper(sweeper *netsweep.Sweeper) ConnectClientOption {
return func(c *ConnectClient) { c.sweeper = sweeper }
}
func NewConnectClient(
ctx context.Context,
config *profilemanager.Config,
statusRecorder *peer.Status,
opts ...ConnectClientOption,
) *ConnectClient {
// Derive the run context here so Stop owns the cancel that unblocks the run
// loop. runCancel is set once at construction, so Stop can call it without
// racing the run loop's startup. Callers therefore need not cancel before Stop.
runCtx, runCancel := context.WithCancel(ctx)
return &ConnectClient{
c := &ConnectClient{
ctx: runCtx,
runCancel: runCancel,
runExited: make(chan struct{}),
@@ -89,6 +115,10 @@ func NewConnectClient(
statusRecorder: statusRecorder,
engineMutex: sync.Mutex{},
}
for _, opt := range opts {
opt(c)
}
return c
}
func (c *ConnectClient) SetUpdateManager(um *updater.Manager) {
@@ -274,6 +304,13 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan
return nil
}
// suspend connection attempts while the OS reports no usable network
if waited, err := c.netState.Wait(c.ctx); err != nil {
return nil
} else if waited {
backOff.Reset()
}
state.Set(StatusConnecting)
engineCtx, cancel := context.WithCancel(c.ctx)
@@ -285,7 +322,8 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan
}()
log.Debugf("connecting to the Management service %s", c.config.ManagementURL.Host)
mgmClient, err := mgm.NewClient(engineCtx, c.config.ManagementURL.Host, myPrivateKey, mgmTlsEnabled)
mgmClient, err := mgm.NewClient(engineCtx, c.config.ManagementURL.Host, myPrivateKey, mgmTlsEnabled,
mgm.WithNetworkState(c.netState), mgm.WithSweeper(c.sweeper))
if err != nil {
// On daemon shutdown / Down() the parent context is cancelled
// and the dial fails with "context canceled". Wrapping that
@@ -360,7 +398,7 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan
}()
// with the global Netbird config in hand connect (just a connection, no stream yet) Signal
signalClient, err := connectToSignal(engineCtx, loginResp.GetNetbirdConfig(), myPrivateKey)
signalClient, err := connectToSignal(engineCtx, loginResp.GetNetbirdConfig(), myPrivateKey, c.netState, c.sweeper)
if err != nil {
log.Error(err)
return wrapErr(err)
@@ -396,7 +434,8 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan
engineConfig.StateDir = filepath.Dir(path)
}
relayManager := relayClient.NewManager(engineCtx, relayURLs, myPrivateKey.PublicKey().String(), engineConfig.MTU)
relayManager := relayClient.NewManager(engineCtx, relayURLs, myPrivateKey.PublicKey().String(), engineConfig.MTU,
relayClient.WithNetworkState(c.netState), relayClient.WithSweeper(c.sweeper))
c.statusRecorder.SetRelayMgr(relayManager)
if len(relayURLs) > 0 {
if token != nil {
@@ -424,6 +463,7 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan
UpdateManager: c.updateManager,
ClientMetrics: c.clientMetrics,
MetricsCtx: c.ctx,
NetState: c.netState,
}, mobileDependency)
engine.SetSyncResponsePersistence(c.persistSyncResponse)
c.engine = engine
@@ -480,6 +520,16 @@ func (c *ConnectClient) run(mobileDependency MobileDependency, runningChan chan
// status stream stuck at Connecting.
err = backoff.Retry(operation, backoff.WithContext(backOff, c.ctx))
if err != nil {
// Once the client context is cancelled backoff.WithContext surfaces the
// bare context error, and any attempt torn down mid-flight reports the
// same. That cancellation is the caller asking us to stop (Stop, Down or
// an engine restart), so exit cleanly instead of handing back a failure
// the caller would have to distinguish from a real one.
if c.ctx.Err() != nil && errors.Is(err, context.Canceled) {
log.Info("exiting client retry loop, context cancelled")
return nil
}
log.Debugf("exiting client retry loop due to unrecoverable error: %s", err)
if s, ok := gstatus.FromError(err); ok && (s.Code() == codes.PermissionDenied) {
state.Set(StatusNeedsLogin)
@@ -675,7 +725,7 @@ func selectMTU(localMTU uint16, peerMTU int32) uint16 {
}
// connectToSignal creates Signal Service client and established a connection
func connectToSignal(ctx context.Context, wtConfig *mgmProto.NetbirdConfig, ourPrivateKey wgtypes.Key) (*signal.GrpcClient, error) {
func connectToSignal(ctx context.Context, wtConfig *mgmProto.NetbirdConfig, ourPrivateKey wgtypes.Key, netState *netstate.State, sweeper *netsweep.Sweeper) (*signal.GrpcClient, error) {
var sigTLSEnabled bool
if wtConfig.Signal.Protocol == mgmProto.HostConfig_HTTPS {
sigTLSEnabled = true
@@ -683,7 +733,8 @@ func connectToSignal(ctx context.Context, wtConfig *mgmProto.NetbirdConfig, ourP
sigTLSEnabled = false
}
signalClient, err := signal.NewClient(ctx, wtConfig.Signal.Uri, ourPrivateKey, sigTLSEnabled)
signalClient, err := signal.NewClient(ctx, wtConfig.Signal.Uri, ourPrivateKey, sigTLSEnabled,
signal.WithNetworkState(netState), signal.WithSweeper(sweeper))
if err != nil {
log.Errorf("error while connecting to the Signal Exchange Service %s: %s", wtConfig.Signal.Uri, err)
return nil, gstatus.Errorf(codes.FailedPrecondition, "failed connecting to Signal Service : %s", err)

99
client/internal/debug/debug.go
View File

@@ -34,9 +34,8 @@ import (
"github.com/netbirdio/netbird/shared/netiputil"
)
const readmeContent = `Netbird debug bundle
This debug bundle contains the following files.
If the --anonymize flag is set, the files are anonymized to protect sensitive information.
const readmeContent = `This debug bundle contains the following files.
If anonymization is enabled (--anonymize / --anonymize-level), the files are anonymized to protect sensitive information.
status.txt: Anonymized status information of the NetBird client.
client.log: Most recent, anonymized client log file of the NetBird client.
@@ -52,6 +51,7 @@ nftables.txt: Anonymized nftables rules with packet counters across all families
sysctls.txt: Forwarding, reverse-path filter, source-validation, and conntrack accounting sysctl values that the NetBird client may read or modify, if --system-info flag was provided (Linux only).
resolv.conf: DNS resolver configuration from /etc/resolv.conf (Unix systems only), if --system-info flag was provided.
scutil_dns.txt: DNS configuration from scutil --dns (macOS only), if --system-info flag was provided.
dns_windows.txt: Anonymized NRPT rules and policy table in effect, DNS client policy, and per-interface and per-adapter DNS configuration (Windows only), if --system-info flag was provided.
resolved_domains.txt: Anonymized resolved domain IP addresses from the status recorder.
config.txt: Anonymized configuration information of the NetBird client.
network_map.json: Anonymized sync response containing peer configurations, routes, DNS settings, and firewall rules.
@@ -70,21 +70,34 @@ capture.pcap: Packet capture in pcap format. Only present when capture was runni
Anonymization Process
The files in this bundle have been anonymized to protect sensitive information. Here's how the anonymization was applied:
The files in this bundle have been anonymized to protect sensitive information. The level applied to this bundle is recorded at the top of this file. Here's how the anonymization was applied:
IP Addresses
IPv4 addresses are replaced with addresses starting from 198.51.100.0
IPv6 addresses are replaced with addresses starting from 100::
Default level:
- Public IPv4 addresses are replaced with addresses starting from 198.51.100.0
- Public IPv6 addresses are replaced with addresses starting from 2001:db8:ffff::
- IPv6 unique local addresses (fc00::/7) are anonymized as well: their random global ID uniquely identifies the network.
- IP addresses from internal IPv4 ranges and well-known addresses are not anonymized (e.g. 8.8.8.8, 100.64.0.0/10, addresses starting with 192.168., 172.16., 10., 169.254., fe80::).
Strict level (--anonymize-level strict), in addition to the default level:
- Private (RFC 1918), CGNAT (100.64.0.0/10), and link-local (169.254.0.0/16, fe80::/10) addresses are anonymized too.
- Internal IPv4 addresses are replaced with addresses starting from 198.18.0.0 and internal IPv6 addresses with addresses starting from 2001:db8:1::, so internal addresses remain distinguishable from public ones.
- Addresses are mapped in order of first appearance: subnet structure, allocation scheme, and gateway conventions are not preserved. Prefix lengths of networks are preserved.
- Peer names in front of NetBird domains are replaced with numbered placeholders (e.g. peer-1.netbird.cloud), and subdomain labels of other domains with host-N placeholders.
- WireGuard public keys are replaced with consistent placeholder keys.
IP addresses from non public ranges and well known addresses are not anonymized (e.g. 8.8.8.8, 100.64.0.0/10, addresses starting with 192.168., 172.16., 10., etc.).
Reoccuring IP addresses are replaced with the same anonymized address.
Note: The anonymized IP addresses in the status file do not match those in the log and routes files. However, the anonymized IP addresses are consistent within the status file and across the routes and log files.
MAC Addresses
MAC addresses are replaced at every anonymization level with consistent placeholders counting up from 02:00:00:00:00:01. Broadcast, multicast, and all-zero addresses are kept. At the default level a preserved IPv6 link-local address may still embed a MAC address (EUI-64); the strict level anonymizes those addresses.
Domains
All domain names (except for the netbird domains) are replaced with randomly generated strings ending in ".domain". Anonymized domains are consistent across all files in the bundle.
Reoccuring domain names are replaced with the same anonymized domain.
At the strict level, the peer name labels in front of netbird domains are anonymized as well.
Sync Response
The network_map.json file contains the following anonymized information:
@@ -225,6 +238,13 @@ scutil_dns.txt (macOS only):
- Shows DNS configuration for all network interfaces
- Includes search domains, nameservers, and DNS resolver settings
- All IP addresses and domain names are anonymized
dns_windows.txt (Windows only):
- Lists the NRPT rules of both policy stores, the local one and the group policy one, marking the rules the client created
- Follows them with the policy table the resolver has loaded, which differs from the rules while a change has not been picked up yet
- Includes the DNS client group policy, the global TCP/IP and Dnscache parameters, and the DNS values of every interface that has any
- Ends with the resolver configuration in effect per adapter, from GetAdaptersAddresses
- All IP addresses and domain names are anonymized
`
const (
@@ -281,6 +301,7 @@ type BundleGenerator struct {
cliVersion string
anonymize bool
anonymizeLevel anonymize.Level
includeSystemInfo bool
logFileCount uint32
@@ -288,7 +309,10 @@ type BundleGenerator struct {
}
type BundleConfig struct {
Anonymize bool
Anonymize bool
// AnonymizeLevel selects how much the anonymizer redacts.
// anonymize.LevelStrict implies Anonymize.
AnonymizeLevel anonymize.Level
IncludeSystemInfo bool
LogFileCount uint32
}
@@ -327,8 +351,11 @@ func NewBundleGenerator(deps GeneratorDependencies, cfg BundleConfig) *BundleGen
uiLogOpener = openLogFile
}
anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses())
anonymizer.SetLevel(cfg.AnonymizeLevel)
return &BundleGenerator{
anonymizer: anonymize.NewAnonymizer(anonymize.DefaultAddresses()),
anonymizer: anonymizer,
internalConfig: deps.InternalConfig,
statusRecorder: deps.StatusRecorder,
@@ -345,7 +372,8 @@ func NewBundleGenerator(deps GeneratorDependencies, cfg BundleConfig) *BundleGen
daemonVersion: deps.DaemonVersion,
cliVersion: deps.CliVersion,
anonymize: cfg.Anonymize,
anonymize: cfg.Anonymize || cfg.AnonymizeLevel >= anonymize.LevelStrict,
anonymizeLevel: cfg.AnonymizeLevel,
includeSystemInfo: cfg.IncludeSystemInfo,
logFileCount: logFileCount,
}
@@ -485,7 +513,13 @@ func (g *BundleGenerator) addSystemInfo() {
}
func (g *BundleGenerator) addReadme() error {
readmeReader := strings.NewReader(readmeContent)
level := "none (anonymization disabled)"
if g.anonymize {
level = g.anonymizeLevel.String()
}
header := fmt.Sprintf("Netbird debug bundle\nAnonymization level applied to this bundle: %s\n", level)
readmeReader := strings.NewReader(header + readmeContent)
if err := g.addFileToZip(readmeReader, "README.txt"); err != nil {
return fmt.Errorf("add README file to zip: %w", err)
}
@@ -507,9 +541,10 @@ func (g *BundleGenerator) addStatus() error {
fullStatus := g.statusRecorder.GetFullStatus()
protoFullStatus := nbstatus.ToProtoFullStatus(fullStatus)
overview := nbstatus.ConvertToStatusOutputOverview(protoFullStatus, nbstatus.ConvertOptions{
Anonymize: g.anonymize,
ProfileName: profName,
DaemonVersion: g.daemonVersion,
Anonymize: g.anonymize,
AnonymizeLevel: g.anonymizeLevel,
ProfileName: profName,
DaemonVersion: g.daemonVersion,
})
overview.CliVersion = g.cliVersion
statusOutput := overview.FullDetailSummary()
@@ -662,7 +697,7 @@ func (g *BundleGenerator) addCommonConfigFields(configContent *strings.Builder)
configContent.WriteString("NetBird Client Configuration:\n\n")
if key, err := wgtypes.ParseKey(g.internalConfig.PrivateKey); err == nil {
configContent.WriteString(fmt.Sprintf("PublicKey: %s\n", key.PublicKey().String()))
configContent.WriteString(fmt.Sprintf("PublicKey: %s\n", g.anonymizer.AnonymizeWGKey(key.PublicKey().String())))
}
configContent.WriteString(fmt.Sprintf("WgIface: %s\n", g.internalConfig.WgIface))
configContent.WriteString(fmt.Sprintf("WgPort: %d\n", g.internalConfig.WgPort))
@@ -958,6 +993,11 @@ func (g *BundleGenerator) addUpdateLogs() error {
}
baseName := filepath.Base(logFile)
data, err = g.anonymizeBytes(data)
if err != nil {
log.Warnf("skipping update log file %s: %v", baseName, err)
continue
}
if err := g.addFileToZip(bytes.NewReader(data), filepath.Join("update-logs", baseName)); err != nil {
return fmt.Errorf("add update log file %s to zip: %w", baseName, err)
}
@@ -985,6 +1025,13 @@ func (g *BundleGenerator) addCorruptedStateFiles() error {
}
fileName := filepath.Base(match)
// Corrupted state files usually fail structured JSON anonymization,
// so run them through the string anonymizer instead.
data, err = g.anonymizeBytes(data)
if err != nil {
log.Warnf("skipping corrupted state file %s: %v", fileName, err)
continue
}
if err := g.addFileToZip(bytes.NewReader(data), "corrupted_states/"+fileName); err != nil {
log.Warnf("Failed to add corrupted state file %s to zip: %v", fileName, err)
continue
@@ -996,6 +1043,27 @@ func (g *BundleGenerator) addCorruptedStateFiles() error {
return nil
}
// anonymizeBytes runs raw file content through the string anonymizer line by
// line when anonymization is enabled. It errors instead of returning partial
// content, so a caller never adds an unanonymized fallback to the bundle.
func (g *BundleGenerator) anonymizeBytes(data []byte) ([]byte, error) {
if !g.anonymize {
return data, nil
}
var buf bytes.Buffer
scanner := bufio.NewScanner(bytes.NewReader(data))
scanner.Buffer(make([]byte, 1024*1024), 1024*1024)
for scanner.Scan() {
buf.WriteString(g.anonymizer.AnonymizeString(scanner.Text()))
buf.WriteByte('\n')
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("anonymize content: %w", err)
}
return buf.Bytes(), nil
}
func (g *BundleGenerator) addMetrics() error {
if g.clientMetrics == nil {
log.Debugf("skipping metrics in debug bundle: no metrics collector")
@@ -1468,6 +1536,7 @@ func anonymizeRemotePeer(peer *mgmProto.RemotePeerConfig, anonymizer *anonymize.
}
peer.Fqdn = anonymizer.AnonymizeDomain(peer.Fqdn)
peer.WgPubKey = anonymizer.AnonymizeWGKey(peer.WgPubKey)
anonymizeSSHConfig(peer.SshConfig)
}

4
client/internal/debug/debug_linux.go
View File

@@ -844,6 +844,10 @@ func collectSysctls() string {
[]string{"net.ipv4.conf.all.src_valid_mark", "net.ipv4.conf.default.src_valid_mark"},
listInterfaceSysctls("ipv4", "src_valid_mark")...,
))
writeSysctlGroup(&builder, "accept_ra", append(
[]string{"net.ipv6.conf.all.accept_ra", "net.ipv6.conf.default.accept_ra"},
listInterfaceSysctls("ipv6", "accept_ra")...,
))
writeSysctlGroup(&builder, "conntrack", []string{
"net.netfilter.nf_conntrack_acct",
"net.netfilter.nf_conntrack_tcp_loose",

2
client/internal/debug/debug_nonunix.go
View File

@@ -1,4 +1,4 @@
//go:build !unix
//go:build !unix && !windows
package debug

443
client/internal/debug/debug_windows.go Normal file
View File

@@ -0,0 +1,443 @@
//go:build windows
package debug
import (
"encoding/hex"
"errors"
"fmt"
"net/netip"
"strings"
"unsafe"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/windows"
"golang.org/x/sys/windows/registry"
nbdns "github.com/netbirdio/netbird/client/internal/dns"
)
const dnsInfoFileName = "dns_windows.txt"
const (
gpoDNSClientRoot = `SOFTWARE\Policies\Microsoft\Windows NT\DNSClient`
tcpipParamsPath = `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters`
dnscacheParams = `SYSTEM\CurrentControlSet\Services\Dnscache\Parameters`
)
// interfaceDNSValues are the per-interface values that decide how a name is
// resolved and registered. Everything the DNS host manager writes is in here,
// so a bundle shows both what we set and what it replaced.
var interfaceDNSValues = []string{
"NameServer",
"DhcpNameServer",
"Domain",
"DhcpDomain",
"SearchList",
"RegistrationEnabled",
"DisableDynamicUpdate",
"MaxNumberOfAddressesToRegister",
"EnableDHCP",
}
// addDNSInfo collects and adds DNS configuration information to the archive
func (g *BundleGenerator) addDNSInfo() error {
if err := g.addFileToZip(strings.NewReader(g.collectDNSInfo()), dnsInfoFileName); err != nil {
return fmt.Errorf("add DNS info to zip: %w", err)
}
return nil
}
// collectDNSInfo renders the report. Everything below it reaches the platform
// through COM and through lazily resolved procedures, which panic when a
// procedure is missing rather than returning an error, and a debug bundle is not
// allowed to take the daemon down. The panic is contained here, and whatever was
// collected before it is kept and reported with it.
func (g *BundleGenerator) collectDNSInfo() (content string) {
var sb strings.Builder
defer func() {
if r := recover(); r != nil {
log.Errorf("collecting Windows DNS configuration panicked: %v", r)
fmt.Fprintf(&sb, "\nerror: collection stopped: %v\n", r)
}
content = sb.String()
}()
sb.WriteString("Windows DNS configuration\n")
sb.WriteString("=========================\n")
adapters, adaptersErr := adapterAddresses()
g.writeNRPTRules(&sb, "NRPT rules, local policy store", nbdns.DNSPolicyConfigRoot)
g.writeNRPTRules(&sb, "NRPT rules, group policy store", nbdns.GPODNSPolicyConfigRoot)
g.writeEffectiveNRPTPolicies(&sb)
g.writeRegistryKey(&sb, "DNS client group policy", gpoDNSClientRoot)
g.writeRegistryKey(&sb, "Global TCP/IP parameters", tcpipParamsPath)
g.writeRegistryKey(&sb, "Dnscache parameters", dnscacheParams)
g.writeInterfaceDNS(&sb, "Per-interface DNS, IPv4", nbdns.InterfaceConfigPath, adapterNames(adapters))
g.writeInterfaceDNS(&sb, "Per-interface DNS, IPv6", nbdns.InterfaceConfigPathV6, adapterNames(adapters))
g.writeAdapterDNS(&sb, adapters, adaptersErr)
return sb.String()
}
// writeNRPTRules lists every rule in a policy store, ours and any other
// product's, since a foreign rule for the same namespace decides resolution
// just as ours does. Rules the client wrote are marked.
func (g *BundleGenerator) writeNRPTRules(sb *strings.Builder, title, root string) {
writeSection(sb, title, root)
names, err := subKeyNames(root)
if err != nil {
fmt.Fprintf(sb, "error: %v\n", err)
return
}
if len(names) == 0 {
sb.WriteString("no rules\n")
return
}
for _, name := range names {
owner := ""
if strings.HasPrefix(strings.ToLower(name), strings.ToLower(nbdns.NRPTKeyPrefix)) {
owner = " (netbird)"
}
fmt.Fprintf(sb, "%s%s\n", name, owner)
g.writeValues(sb, root+`\`+name, nil, " ")
}
}
// writeEffectiveNRPTPolicies reports the table the resolver answers from, which
// the registry cannot show: a rule is written before it is loaded, and it keeps
// being enforced after its key is gone until the resolver reloads its policy.
func (g *BundleGenerator) writeEffectiveNRPTPolicies(sb *strings.Builder) {
writeSection(sb, "NRPT policy table in effect", nrptPolicyClass+"."+nrptPolicyMethod+" in "+nrptPolicyNamespace)
entries, err := effectiveNRPTPolicies()
if err != nil {
fmt.Fprintf(sb, "error: %v\n", err)
return
}
if len(entries) == 0 {
sb.WriteString("no policies\n")
return
}
for _, entry := range entries {
fmt.Fprintf(sb, "%s\n", g.anonymizeValue("Namespace", entry.namespace))
for _, value := range entry.values {
fmt.Fprintf(sb, " %s: %s\n", value.name, g.anonymizeValue(value.name, value.value))
}
}
}
// writeInterfaceDNS reports the DNS values of every interface that has any, so
// the netbird interface can be compared against the physical ones. The registry
// keys the values by GUID, so each is named from the adapter list; a GUID with
// no adapter is a leftover key of an interface that no longer exists.
func (g *BundleGenerator) writeInterfaceDNS(sb *strings.Builder, title, root string, names map[string]string) {
writeSection(sb, title, root)
guids, err := subKeyNames(root)
if err != nil {
fmt.Fprintf(sb, "error: %v\n", err)
return
}
var reported int
for _, guid := range guids {
var iface strings.Builder
g.writeValues(&iface, root+`\`+guid, interfaceDNSValues, " ")
if iface.Len() == 0 {
continue
}
name, ok := names[strings.ToLower(guid)]
if !ok {
name = "no adapter with this GUID"
}
reported++
fmt.Fprintf(sb, "%s (%s)\n%s", guid, name, iface.String())
}
if reported == 0 {
sb.WriteString("no interface holds DNS values\n")
}
}
// writeRegistryKey reports the values of a single key, without its subkeys.
func (g *BundleGenerator) writeRegistryKey(sb *strings.Builder, title, path string) {
writeSection(sb, title, path)
var values strings.Builder
g.writeValues(&values, path, nil, "")
if values.Len() == 0 {
sb.WriteString("no values\n")
return
}
sb.WriteString(values.String())
}
// writeValues renders the values of a key. A nil names list reports every
// value, otherwise only those named and present.
func (g *BundleGenerator) writeValues(sb *strings.Builder, path string, names []string, indent string) {
k, err := registry.OpenKey(registry.LOCAL_MACHINE, path, registry.QUERY_VALUE)
switch {
case errors.Is(err, registry.ErrNotExist), errors.Is(err, windows.ERROR_PATH_NOT_FOUND):
// an absent key is the normal state for the GPO store and for
// interfaces without DNS settings
log.Debugf("HKEY_LOCAL_MACHINE\\%s does not exist", path)
return
case err != nil:
fmt.Fprintf(sb, "%serror: open HKEY_LOCAL_MACHINE\\%s: %v\n", indent, path, err)
return
}
defer closeKey(k)
if names == nil {
names, err = k.ReadValueNames(-1)
if err != nil {
fmt.Fprintf(sb, "%serror: read value names: %v\n", indent, err)
return
}
}
for _, name := range names {
value, err := readRegistryValue(k, name)
switch {
case errors.Is(err, registry.ErrNotExist):
// the caller asks for a fixed set of values, most of which a
// given interface does not carry
continue
case err != nil:
// report rather than omit: a value that is there but cannot be
// read reads as unset otherwise
fmt.Fprintf(sb, "%s%s: error: %v\n", indent, name, err)
continue
}
fmt.Fprintf(sb, "%s%s: %s\n", indent, name, g.anonymizeValue(name, value))
}
}
// anonymizeValue redacts a registry value according to what its name says it
// holds. Domains and addresses are handled per entry rather than by the string
// pass: the pass only replaces domains something else in the bundle already
// seeded, and its address regex would eat the digit labels of a reverse zone.
func (g *BundleGenerator) anonymizeValue(name, value string) string {
if !g.anonymize || value == "" {
return value
}
switch {
case holdsDomains(name):
return joinValueEntries(splitValueEntries(value), g.anonymizeDomain)
case holdsAddresses(name):
return joinValueEntries(splitValueEntries(value), g.anonymizer.AnonymizeIPString)
default:
return g.anonymizer.AnonymizeString(value)
}
}
// holdsDomains reports whether a value name holds domains: the domain list of
// an NRPT rule (Name) or of the policy table (Namespace), a search list, the
// DNS suffix values of the TCP/IP and policy keys, which all end in "Domain"
// (Domain, DhcpDomain, NV Domain, ICSDomain), and a proxy host name.
func holdsDomains(name string) bool {
lower := strings.ToLower(name)
return lower == "name" || lower == "namespace" || lower == "searchlist" ||
strings.HasSuffix(lower, "domain") || strings.HasSuffix(lower, "proxyname")
}
// holdsAddresses reports whether a value name holds DNS server addresses
// (NameServer, DhcpNameServer, GenericDNSServers, NameServers).
func holdsAddresses(name string) bool {
lower := strings.ToLower(name)
return strings.Contains(lower, "nameserver") || strings.Contains(lower, "dnsserver")
}
// adapterNames maps adapter GUIDs, as the registry keys the interfaces, to the
// names an operator sees.
func adapterNames(adapters []*windows.IpAdapterAddresses) map[string]string {
names := make(map[string]string, len(adapters))
for _, adapter := range adapters {
guid := windows.BytePtrToString(adapter.AdapterName)
names[strings.ToLower(guid)] = windows.UTF16PtrToString(adapter.FriendlyName)
}
return names
}
// writeAdapterDNS reports the resolver configuration in effect per adapter,
// which is what the resolver uses for a name no NRPT rule matches.
func (g *BundleGenerator) writeAdapterDNS(sb *strings.Builder, adapters []*windows.IpAdapterAddresses, err error) {
writeSection(sb, "Adapter DNS configuration", "GetAdaptersAddresses")
if err != nil {
fmt.Fprintf(sb, "error: %v\n", err)
return
}
for _, adapter := range adapters {
name := windows.UTF16PtrToString(adapter.FriendlyName)
suffix := g.anonymizeDomain(windows.UTF16PtrToString(adapter.DnsSuffix))
fmt.Fprintf(sb, "%s (index %d, oper status %d)\n", name, adapter.IfIndex, adapter.OperStatus)
fmt.Fprintf(sb, " DNS suffix: %s\n", suffix)
var servers []string
for server := adapter.FirstDnsServerAddress; server != nil; server = server.Next {
addr, ok := netip.AddrFromSlice(server.Address.IP())
if !ok {
continue
}
addr = addr.Unmap()
if g.anonymize {
addr = g.anonymizer.AnonymizeIP(addr)
}
servers = append(servers, addr.String())
}
fmt.Fprintf(sb, " DNS servers: %s\n", strings.Join(servers, ", "))
}
}
// anonymizeDomain anonymizes a single domain, keeping the leading dot an NRPT
// match domain carries.
func (g *BundleGenerator) anonymizeDomain(entry string) string {
if !g.anonymize {
return entry
}
domain, dot := strings.CutPrefix(entry, ".")
if domain == "" {
return entry
}
anonymized := g.anonymizer.AnonymizeDomain(domain)
if dot {
anonymized = "." + anonymized
}
return anonymized
}
// splitValueEntries splits a registry value that holds a list. The separator
// differs per value: a REG_MULTI_SZ arrives joined with ", ", a SearchList is
// comma separated and a NameServer may use commas or spaces.
func splitValueEntries(value string) []string {
return strings.FieldsFunc(value, func(r rune) bool {
return r == ',' || r == ';' || r == ' ' || r == '\t'
})
}
func joinValueEntries(entries []string, anonymize func(string) string) string {
for i, entry := range entries {
entries[i] = anonymize(entry)
}
return strings.Join(entries, ", ")
}
func writeSection(sb *strings.Builder, title, source string) {
fmt.Fprintf(sb, "\n%s\n%s\n%s\n", title, strings.Repeat("-", len(title)), source)
}
func subKeyNames(root string) ([]string, error) {
k, err := registry.OpenKey(registry.LOCAL_MACHINE, root, registry.ENUMERATE_SUB_KEYS)
if err != nil {
return nil, fmt.Errorf("open HKEY_LOCAL_MACHINE\\%s: %w", root, err)
}
defer closeKey(k)
names, err := k.ReadSubKeyNames(-1)
if err != nil {
return nil, fmt.Errorf("read subkey names: %w", err)
}
return names, nil
}
// readRegistryValue renders a value as text regardless of its type, so an
// unexpected type in a policy key still shows up instead of being dropped.
func readRegistryValue(k registry.Key, name string) (string, error) {
_, valueType, err := k.GetValue(name, nil)
if err != nil {
return "", fmt.Errorf("get value %s: %w", name, err)
}
switch valueType {
case registry.SZ, registry.EXPAND_SZ:
value, _, err := k.GetStringValue(name)
if err != nil {
return "", fmt.Errorf("get string value %s: %w", name, err)
}
return value, nil
case registry.MULTI_SZ:
values, _, err := k.GetStringsValue(name)
if err != nil {
return "", fmt.Errorf("get strings value %s: %w", name, err)
}
return strings.Join(values, ", "), nil
case registry.DWORD, registry.QWORD:
value, _, err := k.GetIntegerValue(name)
if err != nil {
return "", fmt.Errorf("get integer value %s: %w", name, err)
}
return fmt.Sprintf("%d (0x%x)", value, value), nil
case registry.BINARY:
value, _, err := k.GetBinaryValue(name)
if err != nil {
return "", fmt.Errorf("get binary value %s: %w", name, err)
}
return hex.EncodeToString(value), nil
default:
return fmt.Sprintf("<unhandled registry type %d>", valueType), nil
}
}
// adapterAddresses returns the adapter list including DNS servers. The call
// reports the size it needs, so grow the buffer and retry until it fits.
func adapterAddresses() (adapters []*windows.IpAdapterAddresses, err error) {
// GetAdaptersAddresses is resolved on first use and panics when it is
// missing, so this reports it as an error and leaves the rest of the
// report intact.
defer func() {
if r := recover(); r != nil {
adapters, err = nil, fmt.Errorf("GetAdaptersAddresses: %v", r)
}
}()
const flags = windows.GAA_FLAG_SKIP_ANYCAST | windows.GAA_FLAG_SKIP_MULTICAST
size := uint32(15000)
for range 3 {
buf := make([]byte, size)
first := (*windows.IpAdapterAddresses)(unsafe.Pointer(&buf[0]))
err := windows.GetAdaptersAddresses(windows.AF_UNSPEC, flags, 0, first, &size)
if errors.Is(err, windows.ERROR_BUFFER_OVERFLOW) {
continue
}
if err != nil {
return nil, fmt.Errorf("GetAdaptersAddresses: %w", err)
}
for adapter := first; adapter != nil; adapter = adapter.Next {
adapters = append(adapters, adapter)
}
return adapters, nil
}
return nil, fmt.Errorf("GetAdaptersAddresses: buffer kept growing")
}
func closeKey(k registry.Key) {
if err := k.Close(); err != nil {
log.Debugf("close registry key: %v", err)
}
}

146
client/internal/debug/debug_windows_test.go Normal file
View File

@@ -0,0 +1,146 @@
//go:build windows
package debug
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/netbirdio/netbird/client/anonymize"
)
func newDNSValueGenerator(level anonymize.Level) *BundleGenerator {
anonymizer := anonymize.NewAnonymizer(anonymize.DefaultAddresses())
anonymizer.SetLevel(level)
return &BundleGenerator{
anonymize: true,
anonymizeLevel: level,
anonymizer: anonymizer,
}
}
// TestAnonymizeValueByName covers the value kinds of the DNS registry keys. The
// names decide the treatment, because the string pass alone replaces only
// domains another part of the bundle already seeded.
func TestAnonymizeValueByName(t *testing.T) {
tests := []struct {
name string
valueName string
value string
assert func(t *testing.T, got string)
}{
{
name: "NRPT match domains keep the leading dot",
valueName: "Name",
value: ".internal.example.com, .corp.example.org",
assert: func(t *testing.T, got string) {
t.Helper()
for _, entry := range strings.Split(got, ", ") {
assert.True(t, strings.HasPrefix(entry, "."), "entry %q should keep its leading dot", entry)
assert.NotContains(t, entry, "example", "entry %q should not keep the original domain", entry)
}
},
},
{
name: "any value name ending in Domain is treated as a domain",
valueName: "ICSDomain",
value: "mshome.net",
assert: func(t *testing.T, got string) {
t.Helper()
assert.NotContains(t, got, "mshome", "should anonymize a domain suffix value")
},
},
{
name: "search list is a comma separated domain list",
valueName: "SearchList",
value: "corp.example.com,branch.example.com",
assert: func(t *testing.T, got string) {
t.Helper()
assert.NotContains(t, got, "example", "should anonymize every search domain")
assert.Len(t, strings.Split(got, ", "), 2, "should keep both search domains")
},
},
{
name: "name servers are anonymized as addresses",
valueName: "DhcpNameServer",
value: "203.0.113.10 8.8.8.8",
assert: func(t *testing.T, got string) {
t.Helper()
assert.NotContains(t, got, "203.0.113.10", "should anonymize a public resolver address")
// well-known resolvers stay readable at every level
assert.Contains(t, got, "8.8.8.8", "should keep a well-known resolver address")
},
},
{
name: "opaque values are left to the string pass",
valueName: "DataBasePath",
value: `%SystemRoot%\System32\drivers\etc`,
assert: func(t *testing.T, got string) {
t.Helper()
assert.Equal(t, `%SystemRoot%\System32\drivers\etc`, got, "should not alter a path")
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
g := newDNSValueGenerator(anonymize.LevelDefault)
tc.assert(t, g.anonymizeValue(tc.valueName, tc.value))
})
}
}
// TestParseNRPTPolicyTable parses the MOF text of the policy table out
// parameters, as the provider on a client with one NRPT rule renders it.
func TestParseNRPTPolicyTable(t *testing.T) {
const text = `[abstract]
class __PARAMETERS
{
[Out, EmbeddedInstance("DnsClientPolicyConfiguration"): ToSubClass, ID(2): DisableOverride ToInstance] DnsClientPolicyConfiguration cmdletOutput[] = {
instance of DnsClientPolicyConfiguration
{
DirectAccessProxyType = "NoProxy";
DirectAccessQueryIPsecRequired = FALSE;
NameEncoding = "Utf8WithoutMapping";
Namespace = ".0.100.in-addr.arpa";
},
instance of DnsClientPolicyConfiguration
{
DirectAccessProxyType = "NoProxy";
NameEncoding = "Utf8WithoutMapping";
NameServers = {"100.0.255.254", "100.0.255.253"};
Namespace = ".nb.internal";
}};
[in] boolean Effective;
[out] uint32 ReturnValue = 0;
};
`
entries := parseNRPTPolicyTable(text)
require.Len(t, entries, 2, "should parse both embedded instances")
assert.Equal(t, ".0.100.in-addr.arpa", entries[0].namespace, "should read the namespace of the first instance")
assert.Equal(t, ".nb.internal", entries[1].namespace, "should read the namespace of the second instance")
assert.Equal(t, []registryValue{
{name: "DirectAccessProxyType", value: "NoProxy"},
{name: "DirectAccessQueryIPsecRequired", value: "FALSE"},
{name: "NameEncoding", value: "Utf8WithoutMapping"},
}, entries[0].values, "should keep the remaining values in order")
assert.Contains(t, entries[1].values, registryValue{name: "NameServers", value: "100.0.255.254, 100.0.255.253"},
"should flatten a MOF array")
for _, value := range entries[1].values {
assert.NotContains(t, value.name, "ReturnValue", "should not read the class level parameters as values")
}
}
func TestParseNRPTPolicyTableEmpty(t *testing.T) {
assert.Empty(t, parseNRPTPolicyTable(""), "should parse no entries from empty text")
assert.Empty(t, parseNRPTPolicyTable("class __PARAMETERS\n{\n};\n"), "should parse no entries from a table with no instances")
}

317
client/internal/debug/nrpt_windows.go Normal file
View File

@@ -0,0 +1,317 @@
//go:build windows
package debug
import (
"errors"
"fmt"
"runtime"
"strings"
"time"
"github.com/go-ole/go-ole"
"github.com/go-ole/go-ole/oleutil"
log "github.com/sirupsen/logrus"
)
const (
// The NRPT policy table is reachable through the CIM class that backs
// Get-DnsClientNrptPolicy. Unlike the rules in the registry, the table is
// what the resolver currently has loaded, which is the only way to tell an
// applied rule from one that is merely written, in either direction.
nrptPolicyNamespace = `root\Microsoft\Windows\DNS`
nrptPolicyClass = "PS_DnsClientNrptPolicy"
nrptPolicyMethod = "Get"
// The class has no instances, so the table comes from the out parameters
// of a static method call, rendered as MOF text: the embedded instances
// arrive as a safe array of objects, which cannot be read back through the
// COM bindings, and the text form carries all of them.
nrptPolicyInstanceKeyword = "instance of DnsClientPolicyConfiguration"
nrptPolicyTimeout = 15 * time.Second
)
// COM initialization results that leave the calling thread usable: S_FALSE for
// a thread this process already initialized, RPC_E_CHANGED_MODE for one that
// belongs to another apartment.
const (
sFalse = 0x00000001
rpcEChangedMode = 0x80010106
)
// nrptQueryInFlight admits one read of the policy table at a time. A provider
// that stops answering keeps its goroutine and the OS thread that goroutine
// pinned, so a later bundle reports that instead of pinning another one.
var nrptQueryInFlight = make(chan struct{}, 1)
// nrptPolicyEntry is one namespace of the effective policy table, holding the
// values of an embedded DnsClientPolicyConfiguration instance in the order the
// provider reported them.
type nrptPolicyEntry struct {
namespace string
values []registryValue
}
// registryValue is a name and its rendered value, shared by the registry and
// policy table readers so both anonymize by value name the same way.
type registryValue struct {
name string
value string
}
// effectiveNRPTPolicies reads the effective NRPT table. The call is bounded
// because a WMI provider can block indefinitely and a debug bundle must not.
func effectiveNRPTPolicies() ([]nrptPolicyEntry, error) {
type result struct {
text string
err error
}
select {
case nrptQueryInFlight <- struct{}{}:
default:
return nil, errors.New("an earlier read of the policy table has not returned")
}
done := make(chan result, 1)
go func() {
// the slot is released here rather than by the caller, so a read that
// outlives the timeout holds it until the provider answers
defer func() { <-nrptQueryInFlight }()
text, err := nrptPolicyTableText()
done <- result{text: text, err: err}
}()
select {
case res := <-done:
if res.err != nil {
return nil, res.err
}
return parseNRPTPolicyTable(res.text), nil
case <-time.After(nrptPolicyTimeout):
return nil, errors.New("read of the policy table timed out")
}
}
// nrptPolicyTableText calls the policy table method and returns the MOF text of
// its out parameters.
func nrptPolicyTableText() (text string, err error) {
// COM is per thread, and the collection is short lived, so the thread is
// pinned for the duration rather than initialized for the process.
runtime.LockOSThread()
defer runtime.UnlockOSThread()
defer func() {
// The COM call chain is dynamically typed, so a provider that answers
// with an unexpected shape must not take the daemon down with it.
if r := recover(); r != nil {
err = fmt.Errorf("read NRPT policy table: %v", r)
}
}()
owns, err := coInitialize()
if err != nil {
return "", err
}
if owns {
defer ole.CoUninitialize()
}
locator, err := oleutil.CreateObject("WbemScripting.SWbemLocator")
if err != nil {
return "", fmt.Errorf("create WMI locator: %w", err)
}
defer locator.Release()
dispatch, err := locator.QueryInterface(ole.IID_IDispatch)
if err != nil {
return "", fmt.Errorf("query WMI locator interface: %w", err)
}
defer dispatch.Release()
service, err := dispatchCall(dispatch, "ConnectServer", nil, nrptPolicyNamespace)
if err != nil {
return "", fmt.Errorf("connect to %s: %w", nrptPolicyNamespace, err)
}
defer service.Release()
inParams, err := spawnMethodInParams(service)
if err != nil {
return "", err
}
defer inParams.Release()
// The effective table is the merge of the local and the group policy
// store, which is what the resolver answers from.
if _, err := oleutil.PutProperty(inParams, "Effective", true); err != nil {
return "", fmt.Errorf("set Effective parameter: %w", err)
}
outParams, err := dispatchCall(service, "ExecMethod", nrptPolicyClass, nrptPolicyMethod, inParams)
if err != nil {
return "", fmt.Errorf("call %s.%s: %w", nrptPolicyClass, nrptPolicyMethod, err)
}
defer outParams.Release()
textVariant, err := oleutil.CallMethod(outParams, "GetObjectText_")
if err != nil {
return "", fmt.Errorf("render policy table: %w", err)
}
defer func() {
if err := textVariant.Clear(); err != nil {
log.Debugf("clear policy table variant: %v", err)
}
}()
return textVariant.ToString(), nil
}
// spawnMethodInParams builds the in parameters instance the method needs. The
// provider rejects the call without one, even when every parameter is optional.
func spawnMethodInParams(service *ole.IDispatch) (*ole.IDispatch, error) {
class, err := dispatchCall(service, "Get", nrptPolicyClass)
if err != nil {
return nil, fmt.Errorf("get class %s: %w", nrptPolicyClass, err)
}
defer class.Release()
methods, err := dispatchProperty(class, "Methods_")
if err != nil {
return nil, fmt.Errorf("get class methods: %w", err)
}
defer methods.Release()
method, err := dispatchCall(methods, "Item", nrptPolicyMethod)
if err != nil {
return nil, fmt.Errorf("get method %s: %w", nrptPolicyMethod, err)
}
defer method.Release()
params, err := dispatchProperty(method, "InParameters")
if err != nil {
return nil, fmt.Errorf("get method parameters: %w", err)
}
defer params.Release()
inParams, err := dispatchCall(params, "SpawnInstance_")
if err != nil {
return nil, fmt.Errorf("spawn parameter instance: %w", err)
}
return inParams, nil
}
// parseNRPTPolicyTable pulls the embedded instances out of the MOF text. Each
// instance is a namespace of the table, with one name and value per line.
func parseNRPTPolicyTable(text string) []nrptPolicyEntry {
var entries []nrptPolicyEntry
var current *nrptPolicyEntry
for _, line := range strings.Split(text, "\n") {
line = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(line), ";"))
switch {
case strings.HasPrefix(line, nrptPolicyInstanceKeyword):
entries = append(entries, nrptPolicyEntry{})
current = &entries[len(entries)-1]
continue
case strings.HasPrefix(line, "}"):
// closes an instance, and the array with the last one, so the
// class level parameters that follow are not read as values
current = nil
continue
case current == nil, line == "{":
continue
}
name, value, ok := strings.Cut(line, " = ")
if !ok {
continue
}
value = unquoteMOFValue(value)
if name == "Namespace" {
current.namespace = value
continue
}
current.values = append(current.values, registryValue{name: name, value: value})
}
return entries
}
// unquoteMOFValue renders a MOF scalar or array as plain text: "a" becomes a,
// and {"a", "b"} becomes a, b.
func unquoteMOFValue(value string) string {
value = strings.TrimSpace(value)
if inner, ok := strings.CutPrefix(value, "{"); ok {
value = strings.TrimSuffix(inner, "}")
entries := strings.Split(value, ",")
for i, entry := range entries {
entries[i] = strings.Trim(strings.TrimSpace(entry), `"`)
}
return strings.Join(entries, ", ")
}
return strings.Trim(value, `"`)
}
// coInitialize prepares the calling thread for COM and reports whether this
// call owns the initialization, which decides whether it may be balanced with
// CoUninitialize. S_FALSE took a reference on a thread this process had already
// initialized and so has to be released, while RPC_E_CHANGED_MODE took none:
// the thread belongs to another apartment, which is usable but is not ours to
// uninitialize.
func coInitialize() (bool, error) {
err := ole.CoInitializeEx(0, ole.COINIT_MULTITHREADED)
if err == nil {
return true, nil
}
var oleErr *ole.OleError
if errors.As(err, &oleErr) {
switch oleErr.Code() {
case sFalse:
return true, nil
case rpcEChangedMode:
return false, nil
}
}
return false, fmt.Errorf("initialize COM: %w", err)
}
// dispatchCall calls a COM method that returns an object.
func dispatchCall(dispatch *ole.IDispatch, method string, params ...any) (*ole.IDispatch, error) {
variant, err := oleutil.CallMethod(dispatch, method, params...)
if err != nil {
return nil, err
}
object := variant.ToIDispatch()
if object == nil {
return nil, fmt.Errorf("%s returned no object", method)
}
return object, nil
}
// dispatchProperty reads a COM property that holds an object.
func dispatchProperty(dispatch *ole.IDispatch, property string) (*ole.IDispatch, error) {
variant, err := oleutil.GetProperty(dispatch, property)
if err != nil {
return nil, err
}
object := variant.ToIDispatch()
if object == nil {
return nil, fmt.Errorf("property %s holds no object", property)
}
return object, nil
}

10
client/internal/debug/wgshow.go
View File

@@ -35,14 +35,14 @@ func (g *BundleGenerator) toWGShowFormat(s *configurer.Stats) string {
var sb strings.Builder
sb.WriteString(fmt.Sprintf("interface: %s\n", s.DeviceName))
sb.WriteString(fmt.Sprintf(" public key: %s\n", s.PublicKey))
sb.WriteString(fmt.Sprintf(" public key: %s\n", g.anonymizer.AnonymizeWGKey(s.PublicKey)))
sb.WriteString(fmt.Sprintf(" listen port: %d\n", s.ListenPort))
if s.FWMark != 0 {
sb.WriteString(fmt.Sprintf(" fwmark: %#x\n", s.FWMark))
}
for _, peer := range s.Peers {
sb.WriteString(fmt.Sprintf("\npeer: %s\n", peer.PublicKey))
sb.WriteString(fmt.Sprintf("\npeer: %s\n", g.anonymizer.AnonymizeWGKey(peer.PublicKey)))
if peer.Endpoint.IP != nil {
if g.anonymize {
anonEndpoint := g.anonymizer.AnonymizeUDPAddr(peer.Endpoint)
@@ -54,7 +54,11 @@ func (g *BundleGenerator) toWGShowFormat(s *configurer.Stats) string {
if len(peer.AllowedIPs) > 0 {
var ipStrings []string
for _, ipnet := range peer.AllowedIPs {
ipStrings = append(ipStrings, ipnet.String())
ipStr := ipnet.String()
if g.anonymize {
ipStr = g.anonymizer.AnonymizeIPString(ipStr)
}
ipStrings = append(ipStrings, ipStr)
}
sb.WriteString(fmt.Sprintf(" allowed ips: %s\n", strings.Join(ipStrings, ", ")))
}

87
client/internal/dns/host_darwin.go
View File

@@ -267,18 +267,38 @@ func (s *systemConfigurator) getSystemDNSSettings() (SystemDNSSettings, error) {
return SystemDNSSettings{}, fmt.Errorf("sending the command: %w", err)
}
var dnsSettings SystemDNSSettings
dnsSettings, serverAddresses, err := parseSystemDNSSettings(b)
if err != nil {
return dnsSettings, err
}
s.mu.Lock()
s.origNameservers = serverAddresses
s.mu.Unlock()
return dnsSettings, nil
}
// parseSystemDNSSettings parses the output of `scutil show State:/Network/Service/<id>/DNS`.
// Lines that don't match the expected "index : value" shape are skipped: hosts with unusual
// network services (e.g. orphaned hardware ports) can produce entries without a value.
func parseSystemDNSSettings(out []byte) (SystemDNSSettings, []netip.Addr, error) {
// port is not exposed by scutil, default to 53
dnsSettings := SystemDNSSettings{ServerPort: DefaultPort}
var serverAddresses []netip.Addr
inSearchDomainsArray := false
inServerAddressesArray := false
scanner := bufio.NewScanner(bytes.NewReader(b))
scanner := bufio.NewScanner(bytes.NewReader(out))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
switch {
case strings.HasPrefix(line, "DomainName :"):
domainName := strings.TrimSpace(strings.Split(line, ":")[1])
dnsSettings.Domains = append(dnsSettings.Domains, domainName)
domainName := strings.TrimSpace(strings.TrimPrefix(line, "DomainName :"))
if domainName != "" {
dnsSettings.Domains = append(dnsSettings.Domains, domainName)
}
continue
case line == "SearchDomains : <array> {":
inSearchDomainsArray = true
continue
@@ -288,36 +308,45 @@ func (s *systemConfigurator) getSystemDNSSettings() (SystemDNSSettings, error) {
case line == "}":
inSearchDomainsArray = false
inServerAddressesArray = false
continue
}
if !inSearchDomainsArray && !inServerAddressesArray {
continue
}
parts := strings.SplitN(line, " : ", 2)
if len(parts) != 2 {
log.Debugf("skipping unexpected scutil DNS line %q", line)
continue
}
value := strings.TrimSpace(parts[1])
if value == "" {
continue
}
if inSearchDomainsArray {
searchDomain := strings.Split(line, " : ")[1]
dnsSettings.Domains = append(dnsSettings.Domains, searchDomain)
} else if inServerAddressesArray {
address := strings.Split(line, " : ")[1]
if ip, err := netip.ParseAddr(address); err == nil && !ip.IsUnspecified() {
ip = ip.Unmap()
serverAddresses = append(serverAddresses, ip)
// Prefer the first IPv4 server as ServerIP since our DNS listener is IPv4.
if !dnsSettings.ServerIP.IsValid() && ip.Is4() {
dnsSettings.ServerIP = ip
}
}
dnsSettings.Domains = append(dnsSettings.Domains, value)
continue
}
ip, err := netip.ParseAddr(value)
if err != nil || ip.IsUnspecified() {
continue
}
ip = ip.Unmap()
serverAddresses = append(serverAddresses, ip)
// Prefer the first IPv4 server as ServerIP since our DNS listener is IPv4.
if !dnsSettings.ServerIP.IsValid() && ip.Is4() {
dnsSettings.ServerIP = ip
}
}
if err := scanner.Err(); err != nil {
return dnsSettings, err
return dnsSettings, serverAddresses, err
}
// default to 53 port
dnsSettings.ServerPort = DefaultPort
s.mu.Lock()
s.origNameservers = serverAddresses
s.mu.Unlock()
return dnsSettings, nil
return dnsSettings, serverAddresses, nil
}
func (s *systemConfigurator) getOriginalNameservers() []netip.Addr {
@@ -435,11 +464,15 @@ func (s *systemConfigurator) getPrimaryService() (string, string, error) {
router := ""
for scanner.Scan() {
text := scanner.Text()
parts := strings.SplitN(text, ":", 2)
if len(parts) != 2 {
continue
}
if strings.Contains(text, "PrimaryService") {
primaryService = strings.TrimSpace(strings.Split(text, ":")[1])
primaryService = strings.TrimSpace(parts[1])
}
if strings.Contains(text, "Router") {
router = strings.TrimSpace(strings.Split(text, ":")[1])
router = strings.TrimSpace(parts[1])
}
}
if err := scanner.Err(); err != nil && err != io.EOF {

114
client/internal/dns/host_darwin_test.go
View File

@@ -328,6 +328,120 @@ func removeTestDNSKey(key string) error {
return err
}
func TestParseSystemDNSSettings(t *testing.T) {
tests := []struct {
name string
output string
expectedDomains []string
expectedServers []netip.Addr
expectedIP netip.Addr
}{
{
name: "well_formed",
output: `<dictionary> {
DomainName : example.com
SearchDomains : <array> {
0 : example.com
1 : corp.example.com
}
ServerAddresses : <array> {
0 : 192.168.1.1
1 : fd00::53
}
}
`,
expectedDomains: []string{"example.com", "example.com", "corp.example.com"},
expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1"), netip.MustParseAddr("fd00::53")},
expectedIP: netip.MustParseAddr("192.168.1.1"),
},
{
// entries without a value after the separator used to panic with
// "index out of range [1] with length 1"
name: "malformed_array_entries_skipped",
output: `<dictionary> {
SearchDomains : <array> {
0 :
(null)
1 : corp.example.com
}
ServerAddresses : <array> {
0 :
1 : 192.168.1.1
}
}
`,
expectedDomains: []string{"corp.example.com"},
expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1")},
expectedIP: netip.MustParseAddr("192.168.1.1"),
},
{
name: "domain_name_without_value_skipped",
output: `<dictionary> {
DomainName :
ServerAddresses : <array> {
0 : 192.168.1.1
}
}
`,
expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1")},
expectedIP: netip.MustParseAddr("192.168.1.1"),
},
{
name: "ipv6_first_prefers_ipv4_server_ip",
output: `<dictionary> {
ServerAddresses : <array> {
0 : fd00::53
1 : 192.168.1.1
}
}
`,
expectedServers: []netip.Addr{netip.MustParseAddr("fd00::53"), netip.MustParseAddr("192.168.1.1")},
expectedIP: netip.MustParseAddr("192.168.1.1"),
},
{
name: "invalid_and_unspecified_addresses_skipped",
output: `<dictionary> {
ServerAddresses : <array> {
0 : (null)
1 : 0.0.0.0
2 : 192.168.1.1
}
}
`,
expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1")},
expectedIP: netip.MustParseAddr("192.168.1.1"),
},
{
name: "v4_mapped_address_unmapped",
output: `<dictionary> {
ServerAddresses : <array> {
0 : ::ffff:192.168.1.1
}
}
`,
expectedServers: []netip.Addr{netip.MustParseAddr("192.168.1.1")},
expectedIP: netip.MustParseAddr("192.168.1.1"),
},
{
name: "empty_output",
output: "",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
settings, servers, err := parseSystemDNSSettings([]byte(tc.output))
require.NoError(t, err, "parsing should not fail")
assert.Equal(t, tc.expectedDomains, settings.Domains, "domains should match")
assert.Equal(t, tc.expectedServers, servers, "server addresses should match")
assert.Equal(t, tc.expectedIP, settings.ServerIP, "server IP should match")
assert.Equal(t, DefaultPort, settings.ServerPort, "server port should default to 53")
})
}
}
func TestGetOriginalNameservers(t *testing.T) {
configurator := &systemConfigurator{
createdKeys: make(map[string]struct{}),

132
client/internal/dns/host_windows.go
View File

@@ -31,10 +31,30 @@ var (
dnsFlushResolverCacheFn = dnsapi.NewProc("DnsFlushResolverCache")
)
// Registry locations of the host DNS configuration this package programs,
// exported so a diagnostic reader reports the same locations that are written.
const (
dnsPolicyConfigMatchPath = `SYSTEM\CurrentControlSet\Services\Dnscache\Parameters\DnsPolicyConfig\NetBird-Match`
gpoDnsPolicyRoot = `SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig`
gpoDnsPolicyConfigMatchPath = gpoDnsPolicyRoot + `\NetBird-Match`
// NRPTKeyPrefix starts the name of every NRPT rule key this client creates.
// Older versions used different layouts under the same prefix: a single
// unsuffixed key, then one key per domain, now one key per batch of domains.
NRPTKeyPrefix = "NetBird-Match"
// DNSPolicyConfigRoot holds the NRPT rules of the local policy store.
DNSPolicyConfigRoot = `SYSTEM\CurrentControlSet\Services\Dnscache\Parameters\DnsPolicyConfig`
// GPODNSPolicyConfigRoot holds the NRPT rules of the group policy store,
// which takes precedence over the local one when it is present.
GPODNSPolicyConfigRoot = `SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig`
// InterfaceConfigPath and InterfaceConfigPathV6 hold the per-interface DNS
// settings, keyed by interface GUID, in separate hives per address family.
InterfaceConfigPath = `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces`
InterfaceConfigPathV6 = `SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces`
)
const (
dnsPolicyConfigMatchPath = DNSPolicyConfigRoot + `\` + NRPTKeyPrefix
gpoDnsPolicyConfigMatchPath = GPODNSPolicyConfigRoot + `\` + NRPTKeyPrefix
dnsPolicyConfigVersionKey = "Version"
dnsPolicyConfigVersionValue = 2
@@ -45,8 +65,6 @@ const (
nrptMaxDomainsPerRule = 50
interfaceConfigPath = `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces`
interfaceConfigPathV6 = `SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces`
interfaceConfigNameServerKey = "NameServer"
interfaceConfigDhcpNameSrvKey = "DhcpNameServer"
interfaceConfigSearchListKey = "SearchList"
@@ -73,7 +91,6 @@ type registryConfigurator struct {
guid string
routingAll bool
gpo bool
nrptEntryCount int
origNameservers []netip.Addr
}
@@ -84,7 +101,7 @@ func newHostManager(wgInterface WGIface) (*registryConfigurator, error) {
}
var useGPO bool
k, err := registry.OpenKey(registry.LOCAL_MACHINE, gpoDnsPolicyRoot, registry.QUERY_VALUE)
k, err := registry.OpenKey(registry.LOCAL_MACHINE, GPODNSPolicyConfigRoot, registry.QUERY_VALUE)
if err != nil {
log.Debugf("failed to open GPO DNS policy root: %v", err)
} else {
@@ -123,7 +140,7 @@ func (r *registryConfigurator) captureOriginalNameservers() ([]netip.Addr, error
seen := make(map[netip.Addr]struct{})
var out []netip.Addr
var merr *multierror.Error
for _, root := range []string{interfaceConfigPath, interfaceConfigPathV6} {
for _, root := range []string{InterfaceConfigPath, InterfaceConfigPathV6} {
addrs, err := r.captureFromTcpipRoot(root)
if err != nil {
merr = multierror.Append(merr, fmt.Errorf("%s: %w", root, err))
@@ -306,14 +323,9 @@ func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig, stateManager
}
if len(matchDomains) != 0 {
count, err := r.addDNSMatchPolicy(matchDomains, config.ServerIP)
// Update count even on error to ensure cleanup covers partially created rules
r.nrptEntryCount = count
if err != nil {
if err := r.addDNSMatchPolicy(matchDomains, config.ServerIP); err != nil {
return fmt.Errorf("add dns match policy: %w", err)
}
} else {
r.nrptEntryCount = 0
}
r.updateState(stateManager)
@@ -329,9 +341,8 @@ func (r *registryConfigurator) applyDNSConfig(config HostDNSConfig, stateManager
func (r *registryConfigurator) updateState(stateManager *statemanager.Manager) {
if err := stateManager.UpdateState(&ShutdownState{
Guid: r.guid,
GPO: r.gpo,
NRPTEntryCount: r.nrptEntryCount,
Guid: r.guid,
GPO: r.gpo,
}); err != nil {
log.Errorf("failed to update shutdown state: %s", err)
}
@@ -346,7 +357,7 @@ func (r *registryConfigurator) addDNSSetupForAll(ip netip.Addr) error {
return nil
}
func (r *registryConfigurator) addDNSMatchPolicy(domains []string, ip netip.Addr) (int, error) {
func (r *registryConfigurator) addDNSMatchPolicy(domains []string, ip netip.Addr) error {
// if the gpo key is present, we need to put our DNS settings there, otherwise our config might be ignored
// see https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-gpnrpt/8cc31cb9-20cb-4140-9e85-3e08703b4745
@@ -363,19 +374,17 @@ func (r *registryConfigurator) addDNSMatchPolicy(domains []string, ip netip.Addr
gpoPath := fmt.Sprintf("%s-%d", gpoDnsPolicyConfigMatchPath, ruleIndex)
if err := r.configureDNSPolicy(localPath, batchDomains, ip); err != nil {
return ruleIndex, fmt.Errorf("configure DNS Local policy for rule %d: %w", ruleIndex, err)
return fmt.Errorf("configure DNS Local policy for rule %d: %w", ruleIndex, err)
}
// Increment immediately so the caller's cleanup path knows about this rule
ruleIndex++
if r.gpo {
if err := r.configureDNSPolicy(gpoPath, batchDomains, ip); err != nil {
return ruleIndex, fmt.Errorf("configure gpo DNS policy for rule %d: %w", ruleIndex-1, err)
return fmt.Errorf("configure gpo DNS policy for rule %d: %w", ruleIndex, err)
}
}
log.Debugf("added NRPT rule %d with %d domains", ruleIndex-1, len(batchDomains))
log.Debugf("added NRPT rule %d with %d domains", ruleIndex, len(batchDomains))
ruleIndex++
}
if r.gpo {
@@ -385,7 +394,7 @@ func (r *registryConfigurator) addDNSMatchPolicy(domains []string, ip netip.Addr
}
log.Infof("added %d NRPT rules for %d domains", ruleIndex, len(domains))
return ruleIndex, nil
return nil
}
func (r *registryConfigurator) configureDNSPolicy(policyPath string, domains []string, ip netip.Addr) error {
@@ -450,7 +459,7 @@ func (r *registryConfigurator) flushDNSCache() {
ret, _, err := dnsFlushResolverCacheFn.Call()
if ret == 0 {
if err != nil && !errors.Is(err, syscall.Errno(0)) {
if !errors.Is(err, syscall.Errno(0)) {
log.Errorf("DnsFlushResolverCache failed: %v", err)
return
}
@@ -496,7 +505,7 @@ func (r *registryConfigurator) deleteInterfaceRegistryKeyProperty(propertyKey st
}
func (r *registryConfigurator) getInterfaceRegistryKey() (registry.Key, error) {
regKeyPath := interfaceConfigPath + "\\" + r.guid
regKeyPath := InterfaceConfigPath + "\\" + r.guid
regKey, err := registry.OpenKey(registry.LOCAL_MACHINE, regKeyPath, registry.SET_VALUE)
if err != nil {
return regKey, fmt.Errorf("open HKEY_LOCAL_MACHINE\\%s: %w", regKeyPath, err)
@@ -518,28 +527,28 @@ func (r *registryConfigurator) restoreHostDNS() error {
return nil
}
// removeDNSMatchPolicies deletes every NRPT rule this client may have created,
// from the local and the GPO policy store. The rules are found by enumerating
// the registry, the only authoritative record of what was written. Cleanup must
// not depend on a rule count: the in-memory one is scoped to a single
// registryConfigurator and the persisted one is deleted on every clean
// disconnect, and a rule left behind keeps resolving names over an interface
// that is gone, until reboot discards the volatile key.
func (r *registryConfigurator) removeDNSMatchPolicies() error {
var merr *multierror.Error
// Try to remove the base entries (for backward compatibility)
if err := removeRegistryKeyFromDNSPolicyConfig(dnsPolicyConfigMatchPath); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove local base entry: %w", err))
}
if err := removeRegistryKeyFromDNSPolicyConfig(gpoDnsPolicyConfigMatchPath); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove GPO base entry: %w", err))
}
for i := 0; i < r.nrptEntryCount; i++ {
localPath := fmt.Sprintf("%s-%d", dnsPolicyConfigMatchPath, i)
gpoPath := fmt.Sprintf("%s-%d", gpoDnsPolicyConfigMatchPath, i)
if err := removeRegistryKeyFromDNSPolicyConfig(localPath); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove local entry %d: %w", i, err))
for _, root := range []string{DNSPolicyConfigRoot, GPODNSPolicyConfigRoot} {
names, err := listNRPTRuleKeys(root)
if err != nil {
merr = multierror.Append(merr, fmt.Errorf("list rule keys under %s: %w", root, err))
continue
}
if err := removeRegistryKeyFromDNSPolicyConfig(gpoPath); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove GPO entry %d: %w", i, err))
for _, name := range names {
path := root + `\` + name
if err := removeRegistryKeyFromDNSPolicyConfig(path); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove entry %s: %w", path, err))
}
}
}
@@ -554,6 +563,39 @@ func (r *registryConfigurator) restoreUncleanShutdownDNS() error {
return r.restoreHostDNS()
}
// listNRPTRuleKeys returns the names of our NRPT rule keys under a policy store
// root. An absent root holds nothing to clean up, which is the normal state of
// the GPO store on a machine without DNS Client policy.
func listNRPTRuleKeys(root string) ([]string, error) {
k, err := registry.OpenKey(registry.LOCAL_MACHINE, root, registry.ENUMERATE_SUB_KEYS)
switch {
case errors.Is(err, registry.ErrNotExist), errors.Is(err, syscall.ERROR_PATH_NOT_FOUND):
// the GPO store is absent on a machine without DNS client policy
log.Debugf("HKEY_LOCAL_MACHINE\\%s does not exist", root)
return nil, nil
case err != nil:
// any other failure has to reach the caller: reporting no rules would
// report a successful cleanup while leaving the rules in place
return nil, fmt.Errorf("open HKEY_LOCAL_MACHINE\\%s: %w", root, err)
}
defer closer(k)
names, err := k.ReadSubKeyNames(-1)
if err != nil {
return nil, fmt.Errorf("read subkey names: %w", err)
}
var ruleKeys []string
for _, name := range names {
// registry key names are case insensitive
if strings.HasPrefix(strings.ToLower(name), strings.ToLower(NRPTKeyPrefix)) {
ruleKeys = append(ruleKeys, name)
}
}
return ruleKeys, nil
}
func removeRegistryKeyFromDNSPolicyConfig(regKeyPath string) error {
k, err := registry.OpenKey(registry.LOCAL_MACHINE, regKeyPath, registry.QUERY_VALUE)
if err != nil {
@@ -585,7 +627,7 @@ func refreshGroupPolicy() error {
)
if ret == 0 {
if err != nil && !errors.Is(err, syscall.Errno(0)) {
if !errors.Is(err, syscall.Errno(0)) {
return fmt.Errorf("RefreshPolicyEx failed: %w", err)
}
return fmt.Errorf("RefreshPolicyEx failed")

70
client/internal/dns/host_windows_test.go
View File

@@ -25,7 +25,7 @@ func TestNRPTEntriesCleanupOnConfigChange(t *testing.T) {
// Create a test interface registry key so updateSearchDomains doesn't fail
testGUID := "{12345678-1234-1234-1234-123456789ABC}"
interfacePath := `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces\` + testGUID
interfacePath := InterfaceConfigPath + `\` + testGUID
testKey, _, err := registry.CreateKey(registry.LOCAL_MACHINE, interfacePath, registry.SET_VALUE)
require.NoError(t, err, "Should create test interface registry key")
testKey.Close()
@@ -56,7 +56,7 @@ func TestNRPTEntriesCleanupOnConfigChange(t *testing.T) {
require.NoError(t, err)
// Verify 3 NRPT rules exist
assert.Equal(t, 3, cfg.nrptEntryCount, "Should create 3 NRPT rules for 125 domains")
assert.Equal(t, 3, countNRPTRuleKeys(t), "Should create 3 NRPT rules for 125 domains")
for i := 0; i < 3; i++ {
exists, err := registryKeyExists(fmt.Sprintf("%s-%d", dnsPolicyConfigMatchPath, i))
require.NoError(t, err)
@@ -81,7 +81,7 @@ func TestNRPTEntriesCleanupOnConfigChange(t *testing.T) {
require.NoError(t, err)
// Verify first 2 NRPT rules exist
assert.Equal(t, 2, cfg.nrptEntryCount, "Should create 2 NRPT rules for 75 domains")
assert.Equal(t, 2, countNRPTRuleKeys(t), "Should create 2 NRPT rules for 75 domains")
for i := 0; i < 2; i++ {
exists, err := registryKeyExists(fmt.Sprintf("%s-%d", dnsPolicyConfigMatchPath, i))
require.NoError(t, err)
@@ -106,9 +106,65 @@ func registryKeyExists(path string) (bool, error) {
return true, nil
}
// TestNRPTCleanupWithoutRuleCount verifies that rules written by a previous run
// are removed by a configurator that has no record of how many there are: an
// unclean exit loses the in-memory count and a clean disconnect deletes the
// persisted one, so cleanup cannot depend on either.
func TestNRPTCleanupWithoutRuleCount(t *testing.T) {
if testing.Short() {
t.Skip("skipping registry integration test in short mode")
}
defer cleanupRegistryKeys(t)
cleanupRegistryKeys(t)
testIP := netip.MustParseAddr("100.64.0.1")
// 75 domains produce two indexed rules, as the current layout does
domains := make([]string, 75)
for i := range domains {
domains[i] = fmt.Sprintf(".domain%d.com", i+1)
}
previousRun := &registryConfigurator{}
require.NoError(t, previousRun.addDNSMatchPolicy(domains, testIP))
// the unsuffixed key an older version would have written
require.NoError(t, previousRun.configureDNSPolicy(dnsPolicyConfigMatchPath, []string{".legacy.example.com"}, testIP))
// a policy owned by someone else, which cleanup must not touch
foreignPath := DNSPolicyConfigRoot + `\DnsPolicyConfigTestForeign`
foreignKey, _, err := registry.CreateKey(registry.LOCAL_MACHINE, foreignPath, registry.SET_VALUE)
require.NoError(t, err, "Should create foreign policy key")
foreignKey.Close()
defer func() {
_ = registry.DeleteKey(registry.LOCAL_MACHINE, foreignPath)
}()
require.Equal(t, 3, countNRPTRuleKeys(t), "Should have two indexed rules and the legacy one")
// a configurator that never applied a DNS config, as one built after a
// restart or from a shutdown state without a count is
freshRun := &registryConfigurator{}
require.NoError(t, freshRun.removeDNSMatchPolicies())
assert.Equal(t, 0, countNRPTRuleKeys(t), "Should remove every rule left by the previous run")
exists, err := registryKeyExists(foreignPath)
require.NoError(t, err)
assert.True(t, exists, "Should not remove a policy that is not ours")
}
func countNRPTRuleKeys(t *testing.T) int {
t.Helper()
names, err := listNRPTRuleKeys(DNSPolicyConfigRoot)
require.NoError(t, err, "Should list NRPT rule keys")
return len(names)
}
func cleanupRegistryKeys(*testing.T) {
// Clean up more entries to account for batching tests with many domains
cfg := &registryConfigurator{nrptEntryCount: 20}
cfg := &registryConfigurator{}
_ = cfg.removeDNSMatchPolicies()
}
@@ -125,7 +181,7 @@ func TestNRPTDomainBatching(t *testing.T) {
// Create a test interface registry key so updateSearchDomains doesn't fail
testGUID := "{12345678-1234-1234-1234-123456789ABC}"
interfacePath := `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces\` + testGUID
interfacePath := InterfaceConfigPath + `\` + testGUID
testKey, _, err := registry.CreateKey(registry.LOCAL_MACHINE, interfacePath, registry.SET_VALUE)
require.NoError(t, err, "Should create test interface registry key")
testKey.Close()
@@ -193,7 +249,7 @@ func TestNRPTDomainBatching(t *testing.T) {
require.NoError(t, err)
// Verify that exactly expectedRuleCount rules were created
assert.Equal(t, tc.expectedRuleCount, cfg.nrptEntryCount,
assert.Equal(t, tc.expectedRuleCount, countNRPTRuleKeys(t),
"Should create %d NRPT rules for %d domains", tc.expectedRuleCount, tc.domainCount)
// Verify all expected rules exist

2
client/internal/dns/response_writer_test.go
View File

@@ -4,7 +4,7 @@ import (
"net"
"testing"
"github.com/golang/mock/gomock"
"go.uber.org/mock/gomock"
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
"github.com/miekg/dns"

2
client/internal/dns/server_privileged_test.go
View File

@@ -9,7 +9,7 @@ import (
"os"
"testing"
"github.com/golang/mock/gomock"
"go.uber.org/mock/gomock"
"github.com/miekg/dns"
"github.com/stretchr/testify/assert"
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"

10
client/internal/dns/unclean_shutdown_windows.go
View File

@@ -5,9 +5,8 @@ import (
)
type ShutdownState struct {
Guid string
GPO bool
NRPTEntryCount int
Guid string
GPO bool
}
func (s *ShutdownState) Name() string {
@@ -16,9 +15,8 @@ func (s *ShutdownState) Name() string {
func (s *ShutdownState) Cleanup() error {
manager := &registryConfigurator{
guid: s.Guid,
gpo: s.GPO,
nrptEntryCount: s.NRPTEntryCount,
guid: s.Guid,
gpo: s.GPO,
}
if err := manager.restoreUncleanShutdownDNS(); err != nil {

2
client/internal/dnsfwd/manager.go
View File

@@ -101,7 +101,7 @@ func (m *Manager) Start(fwdEntries []*ForwarderEntry) error {
m.dnsForwarder = NewDNSForwarder(listenAddress, dnsTTL, m.firewall, m.statusRecorder, m.wgIface)
go func() {
if err := m.dnsForwarder.Listen(fwdEntries); err != nil {
if err := m.dnsForwarder.Listen(fwdEntries); err != nil { //nolint:staticcheck
// todo handle close error if it is exists
log.Errorf("failed to start DNS forwarder, err: %v", err)
}

16
client/internal/engine.go
View File

@@ -23,6 +23,7 @@ import (
"golang.zx2c4.com/wireguard/tun/netstack"
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
"github.com/netbirdio/netbird/client/anonymize"
nberrors "github.com/netbirdio/netbird/client/errors"
"github.com/netbirdio/netbird/client/firewall"
"github.com/netbirdio/netbird/client/firewall/firewalld"
@@ -59,6 +60,7 @@ import (
"github.com/netbirdio/netbird/client/internal/syncstore"
"github.com/netbirdio/netbird/client/internal/updater"
"github.com/netbirdio/netbird/client/jobexec"
"github.com/netbirdio/netbird/client/netstate"
cProto "github.com/netbirdio/netbird/client/proto"
"github.com/netbirdio/netbird/client/system"
nbdns "github.com/netbirdio/netbird/dns"
@@ -183,6 +185,9 @@ type EngineServices struct {
UpdateManager *updater.Manager
ClientMetrics *metrics.ClientMetrics
MetricsCtx context.Context
// NetState gates the reconnection loops on OS-reported network
// availability; nil disables gating.
NetState *netstate.State
}
// Engine is a mechanism responsible for reacting on Signal and Management stream events and managing connections to the remote peers.
@@ -206,6 +211,10 @@ type Engine struct {
config *EngineConfig
mobileDep MobileDependency
// netState gates the peer reconnection guards on OS-reported network
// availability; nil disables gating.
netState *netstate.State
// STUNs is a list of STUN servers used by ICE
STUNs []*stun.URI
// TURNs is a list of STUN servers used by ICE
@@ -341,6 +350,7 @@ func NewEngine(
syncMsgMux: &sync.Mutex{},
config: config,
mobileDep: mobileDep,
netState: services.NetState,
STUNs: []*stun.URI{},
TURNs: []*stun.URI{},
networkSerial: 0,
@@ -1400,6 +1410,7 @@ func (e *Engine) handleBundle(params *mgmProto.BundleParameters) (*mgmProto.JobR
bundleJobParams := debug.BundleConfig{
Anonymize: params.Anonymize,
AnonymizeLevel: anonymize.ParseLevel(params.AnonymizeLevel),
IncludeSystemInfo: true,
LogFileCount: uint32(params.LogFileCount),
}
@@ -1911,7 +1922,8 @@ func (e *Engine) createPeerConn(pubKey string, allowedIPs []netip.Prefix, agentV
Addr: e.getRosenpassAddr(),
PermissiveMode: e.config.RosenpassPermissive,
},
ICEConfig: e.createICEConfig(),
ICEConfig: e.createICEConfig(),
NetworkState: e.netState,
}
serviceDependencies := peer.ServiceDependencies{
@@ -2580,7 +2592,7 @@ func (e *Engine) SetCapture(pc device.PacketCapture) error {
}
afc := capture.NewAFPacketCapture(intf.Name(), sess)
if err := afc.Start(); err != nil {
if err := afc.Start(); err != nil { //nolint:staticcheck // always errors on non-Linux builds
return fmt.Errorf("start AF_PACKET capture on %s: %w", intf.Name(), err)
}
e.afpacketCapture = afc

2
client/internal/engine_privileged_test.go
View File

@@ -12,7 +12,7 @@ import (
"testing"
"time"
"github.com/golang/mock/gomock"
"go.uber.org/mock/gomock"
"github.com/google/uuid"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"

10
client/internal/metrics/connection_type.go
View File

@@ -4,11 +4,17 @@ package metrics
type ConnectionType string
const (
// ConnectionTypeICE represents a direct peer-to-peer connection using ICE
ConnectionTypeICE ConnectionType = "ice"
// ConnectionTypeICEP2P represents a direct peer-to-peer connection using ICE
ConnectionTypeICEP2P ConnectionType = "ice_p2p"
// ConnectionTypeICETurn represents an ICE connection through a TURN server
ConnectionTypeICETurn ConnectionType = "ice_turn"
// ConnectionTypeRelay represents a relayed connection
ConnectionTypeRelay ConnectionType = "relay"
// ConnectionTypeUnknown represents a connection with no active transport. It is not pushed.
ConnectionTypeUnknown ConnectionType = "unknown"
)
// String returns the string representation of the connection type

4
client/internal/metrics/influxdb_test.go
View File

@@ -28,7 +28,7 @@ func TestInfluxDBMetrics_RecordAndExport(t *testing.T) {
WgHandshakeSuccess: time.Now().Add(-1 * time.Second),
}
m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICE, false, ts)
m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICEP2P, false, ts)
var buf bytes.Buffer
err := m.Export(&buf)
@@ -60,7 +60,7 @@ func TestInfluxDBMetrics_ExportDeterministicFieldOrder(t *testing.T) {
// Record multiple times and verify consistent field order
for i := 0; i < 10; i++ {
m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICE, false, ts)
m.RecordConnectionStages(context.Background(), agentInfo, "pair123", ConnectionTypeICEP2P, false, ts)
}
var buf bytes.Buffer

21
client/internal/metrics/infra/README.md
View File

@@ -56,14 +56,33 @@ Measurement: `netbird_peer_connection`
Tags:
- `deployment_type`: "cloud" | "selfhosted" | "unknown"
- `connection_type`: "ice" | "relay"
- `connection_type`: "ice_p2p" | "ice_turn" | "relay" (see below)
- `attempt_type`: "initial" | "reconnection"
- `version`: NetBird version string
- `os`: Operating system (linux, darwin, windows, android, ios, etc.)
- `arch`: CPU architecture (amd64, arm64, etc.)
- `peer_id`: anonymised peer identifier (truncated SHA-256 of the WireGuard public key)
- `connection_pair_id`: deterministic identifier for the peer pair, identical on both sides
**Note:** `SignalingReceived` is set when the first offer or answer arrives from the remote peer (in both initial and reconnection paths). It excludes the potentially unbounded wait for the remote peer to come online.
#### `connection_type` values
Derived from the connection priority (`conntype.ConnPriority`) by `metricsConnType` in `client/internal/peer/conn.go`:
| Value | Priority | Traffic is |
|-------|----------|------------|
| `ice_p2p` | `ICEP2P` | direct peer-to-peer |
| `ice_turn` | `ICETurn` | relayed, through a TURN server |
| `relay` | `Relay` | relayed, through a NetBird relay |
| `unknown` | `None` or unrecognised | no active transport — **the sample is not pushed** |
**Direct traffic is `ice_p2p` only.** `ice_turn` is relayed despite being negotiated by ICE, matching `Conn.isRelayed`.
`None` means no transport is active: not established yet, or reset after a relay drop or a peer-state reset. Such a sample cannot be attributed to a transport, so `recordConnectionMetrics` drops it instead of pushing it — `unknown` therefore never appears in the bucket. Connection counts are counts of connections whose transport was known at sampling time.
**Samples recorded before 0.77 used a single `ice` value** which covered `ICEP2P`, `ICETurn` *and* `None`, so historical `ice` samples overstate direct connections by an unknown amount and must not be compared with `ice_p2p`.
### Sync Duration
Measurement: `netbird_sync`

38
client/internal/peer/conn.go
View File

@@ -26,6 +26,7 @@ import (
"github.com/netbirdio/netbird/client/internal/portforward"
"github.com/netbirdio/netbird/client/internal/rosenpass"
"github.com/netbirdio/netbird/client/internal/stdnet"
"github.com/netbirdio/netbird/client/netstate"
"github.com/netbirdio/netbird/route"
relayClient "github.com/netbirdio/netbird/shared/relay/client"
)
@@ -93,6 +94,10 @@ type ConnConfig struct {
// ICEConfig ICE protocol configuration
ICEConfig icemaker.Config
// NetworkState gates the reconnection guard on OS-reported network
// availability; nil disables gating.
NetworkState *netstate.State
}
type Conn struct {
@@ -254,7 +259,7 @@ func (conn *Conn) open(engineCtx context.Context, firstPacket []byte) error {
conn.handshaker.AddICEListener(conn.workerICE.OnNewOffer)
}
conn.guard = guard.NewGuard(conn.Log, conn.isConnectedOnAllWay, conn.config.Timeout, conn.srWatcher)
conn.guard = guard.NewGuard(conn.Log, conn.isConnectedOnAllWay, conn.config.Timeout, conn.srWatcher, conn.config.NetworkState)
conn.wg.Add(1)
go func() {
@@ -307,6 +312,8 @@ func (conn *Conn) Close(signalToRemote bool) {
if conn.wgWatcherCancel != nil {
conn.wgWatcherCancel()
conn.wgWatcher = nil
conn.wgWatcherCancel = nil
}
conn.workerRelay.CloseConn()
if conn.workerICE != nil {
@@ -438,7 +445,7 @@ func (conn *Conn) onICEConnectionIsReady(priority conntype.ConnPriority, iceConn
conn.dumpState.NewLocalProxy()
wgProxy, err = conn.newProxy(iceConnInfo.RemoteConn)
if err != nil {
conn.Log.Errorf("failed to add turn net.Conn to local proxy: %v", err)
conn.Log.Errorf("failed to add relayed net.Conn to local proxy: %v", err)
return
}
ep = wgProxy.EndpointAddr()
@@ -876,9 +883,8 @@ func (conn *Conn) newProxy(remoteConn net.Conn) (wgproxy.Proxy, error) {
}
wgProxy := conn.config.WgConfig.WgInterface.GetProxy()
if err := wgProxy.AddTurnConn(conn.ctx, udpAddr, remoteConn); err != nil {
conn.Log.Errorf("failed to add turn net.Conn to local proxy: %v", err)
return nil, err
if err := wgProxy.AddRelayedConn(conn.ctx, udpAddr, remoteConn); err != nil {
return nil, fmt.Errorf("add relayed conn to proxy: %w", err)
}
return wgProxy, nil
}
@@ -959,12 +965,9 @@ func (conn *Conn) recordConnectionMetrics() {
priority := conn.currentConnPriority
conn.mu.Unlock()
var connType metrics.ConnectionType
switch priority {
case conntype.Relay:
connType = metrics.ConnectionTypeRelay
default:
connType = metrics.ConnectionTypeICE
connType := metricsConnType(priority)
if connType == metrics.ConnectionTypeUnknown {
return
}
// Record metrics with timestamps - duration calculation happens in metrics package
@@ -1065,3 +1068,16 @@ func boolToConnStatus(connected bool) guard.ConnStatus {
}
return guard.ConnStatusDisconnected
}
func metricsConnType(priority conntype.ConnPriority) metrics.ConnectionType {
switch priority {
case conntype.Relay:
return metrics.ConnectionTypeRelay
case conntype.ICETurn:
return metrics.ConnectionTypeICETurn
case conntype.ICEP2P:
return metrics.ConnectionTypeICEP2P
default:
return metrics.ConnectionTypeUnknown
}
}

32
client/internal/peer/conn_test.go
View File

@@ -11,6 +11,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/netbirdio/netbird/client/iface"
"github.com/netbirdio/netbird/client/internal/metrics"
"github.com/netbirdio/netbird/client/internal/peer/conntype"
"github.com/netbirdio/netbird/client/internal/peer/dispatcher"
"github.com/netbirdio/netbird/client/internal/peer/guard"
"github.com/netbirdio/netbird/client/internal/peer/ice"
@@ -386,3 +388,33 @@ func TestConn_onWGDisconnected_NoEscalationWithoutRosenpass(t *testing.T) {
}
assert.Empty(t, disconnected, "escalation must be limited to rosenpass connections")
}
func TestMetricsConnType(t *testing.T) {
tests := []struct {
name string
priority conntype.ConnPriority
expected metrics.ConnectionType
}{
{"relay", conntype.Relay, metrics.ConnectionTypeRelay},
{"ice over turn is relayed, not p2p", conntype.ICETurn, metrics.ConnectionTypeICETurn},
{"direct p2p", conntype.ICEP2P, metrics.ConnectionTypeICEP2P},
{"unset priority is unknown, not p2p", conntype.None, metrics.ConnectionTypeUnknown},
{"unrecognised priority is unknown", conntype.ConnPriority(99), metrics.ConnectionTypeUnknown},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.expected, metricsConnType(tc.priority))
})
}
}
func TestMetricsConnType_RelayedMatchesIsRelayed(t *testing.T) {
for _, priority := range []conntype.ConnPriority{conntype.None, conntype.Relay, conntype.ICETurn, conntype.ICEP2P} {
conn := &Conn{currentConnPriority: priority}
tag := metricsConnType(priority)
relayedTag := tag == metrics.ConnectionTypeRelay || tag == metrics.ConnectionTypeICETurn
assert.Equal(t, conn.isRelayed(), relayedTag,
"priority %s: isRelayed and the %q metric tag must agree", priority, tag)
}
}

42
client/internal/peer/guard/guard.go
View File

@@ -6,6 +6,8 @@ import (
"github.com/cenkalti/backoff/v4"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/netstate"
)
// ConnStatus represents the connection state as seen by the guard.
@@ -31,20 +33,26 @@ type connStatusFunc func() ConnStatus
// - Relayed connection disconnected
// - ICE candidate changes
type Guard struct {
log *log.Entry
isConnectedOnAllWay connStatusFunc
timeout time.Duration
srWatcher *SRWatcher
log *log.Entry
isConnectedOnAllWay connStatusFunc
timeout time.Duration
srWatcher *SRWatcher
// netState gates reconnect attempts on OS-reported network availability;
// nil disables gating.
netState *netstate.State
relayedConnDisconnected chan struct{}
iCEConnDisconnected chan struct{}
}
func NewGuard(log *log.Entry, isConnectedFn connStatusFunc, timeout time.Duration, srWatcher *SRWatcher) *Guard {
// NewGuard creates a reconnection guard for a peer connection. A nil netState
// disables network availability gating.
func NewGuard(log *log.Entry, isConnectedFn connStatusFunc, timeout time.Duration, srWatcher *SRWatcher, netState *netstate.State) *Guard {
return &Guard{
log: log,
isConnectedOnAllWay: isConnectedFn,
timeout: timeout,
srWatcher: srWatcher,
netState: netState,
relayedConnDisconnected: make(chan struct{}, 1),
iCEConnDisconnected: make(chan struct{}, 1),
}
@@ -96,9 +104,16 @@ func (g *Guard) reconnectLoopWithRetry(ctx context.Context, callback func()) {
iceState := &iceRetryState{log: g.log}
defer iceState.reset()
netChanged := g.netState.Changed()
for {
select {
case <-tickerChannel:
// skip attempts while the OS reports no usable network; the
// netChanged case below resumes the loop once it returns
if !g.netState.IsOnline() {
continue
}
switch g.isConnectedOnAllWay() {
case ConnStatusConnected:
// all good, nothing to do
@@ -135,6 +150,23 @@ func (g *Guard) reconnectLoopWithRetry(ctx context.Context, callback func()) {
tickerChannel = ticker.C
iceState.reset()
case <-netChanged:
// Re-arm for the next transition before acting on this one.
netChanged = g.netState.Changed()
if !g.netState.IsOnline() {
continue
}
// Ticks skipped while offline drove the backoff towards its
// maximum without ever attempting, and left the ICE budget
// frozen — possibly in hourly mode. Recover on our own so the
// peer does not depend on a signal or relay event that never
// comes when both stayed up across the outage.
g.log.Debugf("network is back, reset reconnection ticker")
ticker.Stop()
ticker = g.newReconnectTicker(ctx)
tickerChannel = ticker.C
iceState.reset()
case <-ctx.Done():
g.log.Debugf("context is done, stop reconnect loop")
return

2
client/internal/peer/guard/guard_leak_test.go
View File

@@ -15,7 +15,7 @@ import (
func newTestGuard(status connStatusFunc) *Guard {
srw := NewSRWatcher(nil, nil, nil, ice.Config{})
return NewGuard(log.WithField("test", "guard"), status, 50*time.Millisecond, srw)
return NewGuard(log.WithField("test", "guard"), status, 50*time.Millisecond, srw, nil)
}
// countBackoffTickerGoroutines returns how many goroutines are currently sitting

107
client/internal/peer/guard/guard_netstate_test.go Normal file
View File

@@ -0,0 +1,107 @@
package guard
import (
"context"
"sync/atomic"
"testing"
"time"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/internal/peer/ice"
"github.com/netbirdio/netbird/client/netstate"
)
// newTestGuardWithNetState builds a guard with a realistic MaxInterval: the
// backoff must be able to grow well past the outage, as it does in production
// where the timeout is seconds to minutes.
func newTestGuardWithNetState(status connStatusFunc, netState *netstate.State) *Guard {
srw := NewSRWatcher(nil, nil, nil, ice.Config{})
return NewGuard(log.WithField("test", "guard"), status, 30*time.Second, srw, netState)
}
// TestGuard_RecoversAfterOfflineToOnline covers a peer that stays disconnected
// across a network outage while neither signal nor relay reports an event —
// both stayed up, as on a short airplane mode toggle over Wi-Fi.
//
// Every tick taken while offline is skipped, but it still advances the
// exponential backoff, so by the time the network returns the next tick can be
// tens of seconds away. Without an explicit reaction to the transition the
// peer waits out that interval for a recovery that could start immediately.
func TestGuard_RecoversAfterOfflineToOnline(t *testing.T) {
netState := netstate.New()
var attempts atomic.Int32
g := newTestGuardWithNetState(func() ConnStatus { return ConnStatusDisconnected }, netState)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Start from the reconnect ticker (800ms initial interval), the state a
// peer is in after it loses its connection.
go g.Start(ctx, func() { attempts.Add(1) })
g.SetRelayedConnDisconnected()
// Let the backoff climb: 0.8s, 1.6s, 3.2s, 6.4s ... every tick is skipped
// while offline, but each one doubles the wait for the next.
netState.Set(false)
time.Sleep(8 * time.Second)
offlineAttempts := attempts.Load()
if offlineAttempts != 0 {
t.Fatalf("callback ran %d times while offline, want 0", offlineAttempts)
}
netState.Set(true)
// The next organic tick is now several seconds out, so anything within
// this window can only come from reacting to the transition itself.
pollCtx, stopPolling := context.WithTimeout(ctx, 2*time.Second)
defer stopPolling()
select {
case <-pollCtx.Done():
t.Fatal("peer was not retried within 2s of the network coming back, " +
"with neither a signal nor a relay event to fall back on")
case <-pollUntil(pollCtx, func() bool { return attempts.Load() > 0 }):
}
}
// TestGuard_OfflineTransitionDoesNotRetry checks the other direction: going
// offline must not itself trigger an attempt.
func TestGuard_OfflineTransitionDoesNotRetry(t *testing.T) {
netState := netstate.New()
var attempts atomic.Int32
g := newTestGuardWithNetState(func() ConnStatus { return ConnStatusDisconnected }, netState)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go g.Start(ctx, func() { attempts.Add(1) })
netState.Set(false)
time.Sleep(5 * time.Second)
if got := attempts.Load(); got != 0 {
t.Fatalf("callback ran %d times after going offline, want 0", got)
}
}
// pollUntil closes the returned channel once cond holds. It gives up when ctx
// is done, so the polling goroutine never outlives the test that started it.
func pollUntil(ctx context.Context, cond func() bool) <-chan struct{} {
done := make(chan struct{})
go func() {
for {
if cond() {
close(done)
return
}
select {
case <-ctx.Done():
return
case <-time.After(10 * time.Millisecond):
}
}
}()
return done
}

62
client/internal/peer/handshaker.go
View File

@@ -81,14 +81,19 @@ type Handshaker struct {
func NewHandshaker(log *log.Entry, config ConnConfig, signaler *Signaler, ice *WorkerICE, relay *WorkerRelay, metricsStages *MetricsStages) *Handshaker {
h := &Handshaker{
log: log,
config: config,
signaler: signaler,
ice: ice,
relay: relay,
metricsStages: metricsStages,
remoteOffersCh: make(chan OfferAnswer),
remoteAnswerCh: make(chan OfferAnswer),
log: log,
config: config,
signaler: signaler,
ice: ice,
relay: relay,
metricsStages: metricsStages,
// Buffered by one so an offer or answer that arrives between Open launching
// the Listen goroutine and it reaching its receive is held rather than
// dropped. A peer activated by an incoming signal receives the remote's
// message in that window; an unbuffered channel skips it as "receiver not
// ready", and the connection cannot proceed until the remote re-sends.
remoteOffersCh: make(chan OfferAnswer, 1),
remoteAnswerCh: make(chan OfferAnswer, 1),
}
// assume remote supports ICE until we learn otherwise from received offers
h.remoteICESupported.Store(ice != nil)
@@ -162,29 +167,38 @@ func (h *Handshaker) SendOffer() error {
return h.sendOffer()
}
// OnRemoteOffer handles an offer from the remote peer and returns true if the message was accepted, false otherwise
// doesn't block, discards the message if connection wasn't ready
// OnRemoteOffer hands an offer to Listen without blocking, keeping only the most
// recent one if several arrive before Listen reads them.
func (h *Handshaker) OnRemoteOffer(offer OfferAnswer) {
select {
case h.remoteOffersCh <- offer:
return
default:
h.log.Warnf("skipping remote offer message because receiver not ready")
// connection might not be ready yet to receive so we ignore the message
return
}
enqueueLatest(h.remoteOffersCh, offer)
}
// OnRemoteAnswer handles an offer from the remote peer and returns true if the message was accepted, false otherwise
// doesn't block, discards the message if connection wasn't ready
// OnRemoteAnswer hands an answer to Listen without blocking, keeping only the most
// recent one if several arrive before Listen reads them.
func (h *Handshaker) OnRemoteAnswer(answer OfferAnswer) {
enqueueLatest(h.remoteAnswerCh, answer)
}
// enqueueLatest delivers msg on a one-slot channel without blocking. When the slot
// already holds an unread message the older one is discarded in favor of msg, so a
// message arriving before Listen starts reading is held rather than dropped, and
// the newest wins if several arrive first. Safe because there is a single producer
// (the engine loop): after draining the stale value the send always has room.
func enqueueLatest(ch chan OfferAnswer, msg OfferAnswer) {
select {
case h.remoteAnswerCh <- answer:
case ch <- msg:
return
default:
// connection might not be ready yet to receive so we ignore the message
h.log.Warnf("skipping remote answer message because receiver not ready")
return
}
select {
case <-ch:
default:
}
select {
case ch <- msg:
default:
}
}

63
client/internal/peer/handshaker_test.go Normal file
View File

@@ -0,0 +1,63 @@
package peer
import (
"testing"
"time"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
func newTestHandshaker(t *testing.T) *Handshaker {
t.Helper()
// The tests exercise the answer path, whose Listen branch dispatches to the
// relay listener without sending an answer, so no signaler/ICE/relay is needed.
return NewHandshaker(log.WithField("test", t.Name()), ConnConfig{}, nil, nil, nil, nil)
}
// TestHandshakerHoldsSignalArrivingBeforeListen covers the case where a peer is
// activated by an incoming signal: the remote's offer/answer arrives in the same
// step that opens the connection, before the Listen loop starts reading. The
// message must be held rather than dropped, or the connection cannot proceed until
// the remote re-sends. This is the path taken when an eager peer connects to a
// lazily-managed one.
func TestHandshakerHoldsSignalArrivingBeforeListen(t *testing.T) {
h := newTestHandshaker(t)
processed := make(chan *OfferAnswer, 4)
h.AddRelayListener(func(o *OfferAnswer) { processed <- o })
// Delivered before Listen is reading, as when the peer is woken by the remote's
// signal and the message is delivered right after Open.
h.OnRemoteAnswer(OfferAnswer{WgListenPort: 51820})
go h.Listen(t.Context())
select {
case <-processed:
case <-time.After(2 * time.Second):
assert.Fail(t, "remote-answer dispatch: signal delivered before Listen was ready was dropped")
}
}
// TestHandshakerKeepsLatestSignalBeforeListen covers several signals arriving
// before Listen reads: the newest must win (matching the latest-offer contract),
// rather than the first being kept and later ones discarded.
func TestHandshakerKeepsLatestSignalBeforeListen(t *testing.T) {
h := newTestHandshaker(t)
processed := make(chan *OfferAnswer, 4)
h.AddRelayListener(func(o *OfferAnswer) { processed <- o })
h.OnRemoteAnswer(OfferAnswer{WgListenPort: 1111})
h.OnRemoteAnswer(OfferAnswer{WgListenPort: 2222})
go h.Listen(t.Context())
select {
case got := <-processed:
assert.Equal(t, 2222, got.WgListenPort, "remote-answer dispatch: the latest queued signal should be processed")
case <-time.After(2 * time.Second):
assert.Fail(t, "remote-answer dispatch: queued signal was dropped")
}
}

29
client/internal/peer/listener.go
View File

@@ -1,11 +1,40 @@
package peer
// ClientState identifies the client connection state delivered via
// Listener.OnStateChanged.
type ClientState int
// Client states. The numeric values cross the gomobile boundary (the mobile
// bindings re-export them as integer constants), so they are a wire format:
// append new states at the end, never reorder or insert.
const (
ClientStateDisconnected ClientState = iota
ClientStateConnected
ClientStateConnecting
ClientStateDisconnecting
// ClientStateNoNetwork is an overlay state: it is never stored as the
// last notification, only derived from ClientStateConnecting while the
// OS reports no usable network (see notifier.effectiveState).
ClientStateNoNetwork
)
// Listener is a callback type about the NetBird network connection state
type Listener interface {
// OnStateChanged reports every client state transition. New states are
// delivered only through this callback; the per-state callbacks below
// are kept for compatibility and will be removed once all consumers
// have migrated.
OnStateChanged(state ClientState)
// Deprecated: consume OnStateChanged instead.
OnConnected()
// Deprecated: consume OnStateChanged instead.
OnDisconnected()
// Deprecated: consume OnStateChanged instead.
OnConnecting()
// Deprecated: consume OnStateChanged instead.
OnDisconnecting()
OnAddressChanged(string, string)
OnPeersListChanged(int)
}

111
client/internal/peer/notifier.go
View File

@@ -4,31 +4,64 @@ import (
"sync"
)
const (
stateDisconnected = iota
stateConnected
stateConnecting
stateDisconnecting
)
type notifier struct {
// publishLock orders state publication: it is held across computing the
// effective state and handing it to the listener, so a transition cannot
// overtake a newer one and leave the listener on a stale state.
publishLock sync.Mutex
serverStateLock sync.Mutex
listenersLock sync.Mutex
listener Listener
currentClientState bool
lastNotification int
lastNotification ClientState
lastNumberOfPeers int
lastFqdnAddress string
lastIPAddress string
networkAvailable bool
}
func newNotifier() *notifier {
return &notifier{}
return &notifier{
networkAvailable: true,
}
}
// effectiveState maps the computed state to what listeners should see:
// while the OS reports no usable network, "Connecting" would be a lie —
// connection attempts are suspended — so it is reported as NoNetwork.
// Caller must hold serverStateLock.
func (n *notifier) effectiveState(state ClientState) ClientState {
if !n.networkAvailable && state == ClientStateConnecting {
return ClientStateNoNetwork
}
return state
}
// setNetworkAvailable records the OS network availability and re-notifies
// the listener when the flag flips the effective state (Connecting <->
// NoNetwork).
func (n *notifier) setNetworkAvailable(available bool) {
n.publishLock.Lock()
defer n.publishLock.Unlock()
n.serverStateLock.Lock()
if n.networkAvailable == available {
n.serverStateLock.Unlock()
return
}
previous := n.effectiveState(n.lastNotification)
n.networkAvailable = available
current := n.effectiveState(n.lastNotification)
n.serverStateLock.Unlock()
if previous != current {
n.notify(current)
}
}
func (n *notifier) setListener(listener Listener) {
n.serverStateLock.Lock()
lastNotification := n.lastNotification
lastNotification := n.effectiveState(n.lastNotification)
numOfPeers := n.lastNumberOfPeers
fqdnAddress := n.lastFqdnAddress
address := n.lastIPAddress
@@ -52,6 +85,9 @@ func (n *notifier) removeListener() {
}
func (n *notifier) updateServerStates(mgmState bool, signalState bool) {
n.publishLock.Lock()
defer n.publishLock.Unlock()
n.serverStateLock.Lock()
calculatedState := n.calculateState(mgmState, signalState)
@@ -61,43 +97,54 @@ func (n *notifier) updateServerStates(mgmState bool, signalState bool) {
}
n.lastNotification = calculatedState
effective := n.effectiveState(calculatedState)
n.serverStateLock.Unlock()
n.notify(calculatedState)
n.notify(effective)
}
func (n *notifier) clientStart() {
n.publishLock.Lock()
defer n.publishLock.Unlock()
n.serverStateLock.Lock()
n.currentClientState = true
n.lastNotification = stateConnecting
n.lastNotification = ClientStateConnecting
effective := n.effectiveState(ClientStateConnecting)
n.serverStateLock.Unlock()
n.notify(stateConnecting)
n.notify(effective)
}
func (n *notifier) clientStop() {
n.publishLock.Lock()
defer n.publishLock.Unlock()
n.serverStateLock.Lock()
n.currentClientState = false
n.lastNotification = stateDisconnected
n.lastNotification = ClientStateDisconnected
n.serverStateLock.Unlock()
n.notify(stateDisconnected)
n.notify(ClientStateDisconnected)
}
func (n *notifier) clientTearDown() {
n.publishLock.Lock()
defer n.publishLock.Unlock()
n.serverStateLock.Lock()
n.currentClientState = false
n.lastNotification = stateDisconnecting
n.lastNotification = ClientStateDisconnecting
n.serverStateLock.Unlock()
n.notify(stateDisconnecting)
n.notify(ClientStateDisconnecting)
}
func (n *notifier) isServerStateChanged(newState int) bool {
func (n *notifier) isServerStateChanged(newState ClientState) bool {
return n.lastNotification != newState
}
func (n *notifier) notify(state int) {
func (n *notifier) notify(state ClientState) {
n.listenersLock.Lock()
listener := n.listener
n.listenersLock.Unlock()
@@ -109,20 +156,20 @@ func (n *notifier) notify(state int) {
notifyListener(listener, state)
}
func (n *notifier) calculateState(managementConn, signalConn bool) int {
func (n *notifier) calculateState(managementConn, signalConn bool) ClientState {
if managementConn && signalConn {
return stateConnected
return ClientStateConnected
}
if !managementConn && !signalConn && !n.currentClientState {
return stateDisconnected
return ClientStateDisconnected
}
if n.lastNotification == stateDisconnecting {
return stateDisconnecting
if n.lastNotification == ClientStateDisconnecting {
return ClientStateDisconnecting
}
return stateConnecting
return ClientStateConnecting
}
func (n *notifier) peerListChanged(numOfPeers int) {
@@ -159,15 +206,19 @@ func (n *notifier) localAddressChanged(fqdn, address string) {
listener.OnAddressChanged(fqdn, address)
}
func notifyListener(l Listener, state int) {
func notifyListener(l Listener, state ClientState) {
// legacy per-state callbacks; NoNetwork is delivered only via
// OnStateChanged below
switch state {
case stateDisconnected:
case ClientStateDisconnected:
l.OnDisconnected()
case stateConnected:
case ClientStateConnected:
l.OnConnected()
case stateConnecting:
case ClientStateConnecting:
l.OnConnecting()
case stateDisconnecting:
case ClientStateDisconnecting:
l.OnDisconnecting()
}
l.OnStateChanged(state)
}

108
client/internal/peer/notifier_concurrent_test.go Normal file
View File

@@ -0,0 +1,108 @@
package peer
import (
"sync"
"testing"
"time"
)
type recordingListener struct {
mu sync.Mutex
states []ClientState
onState func(ClientState)
}
func (l *recordingListener) OnStateChanged(state ClientState) {
l.mu.Lock()
l.states = append(l.states, state)
hook := l.onState
l.mu.Unlock()
if hook != nil {
hook(state)
}
}
func (l *recordingListener) last() (ClientState, bool) {
l.mu.Lock()
defer l.mu.Unlock()
if len(l.states) == 0 {
return 0, false
}
return l.states[len(l.states)-1], true
}
func (l *recordingListener) snapshot() []ClientState {
l.mu.Lock()
defer l.mu.Unlock()
return append([]ClientState(nil), l.states...)
}
func (l *recordingListener) OnConnected() {}
func (l *recordingListener) OnDisconnected() {}
func (l *recordingListener) OnConnecting() {}
func (l *recordingListener) OnDisconnecting() {}
func (l *recordingListener) OnAddressChanged(string, string) {}
func (l *recordingListener) OnPeersListChanged(int) {}
// TestNotifier_ConcurrentAvailabilityFlipOrdersPublication holds the first
// transition inside the listener callback and flips availability again from
// another goroutine while it is parked. The second flip must not publish
// ahead of the one in flight, otherwise the listener ends up on a state the
// notifier already superseded.
func TestNotifier_ConcurrentAvailabilityFlipOrdersPublication(t *testing.T) {
n := newNotifier()
n.currentClientState = true
n.lastNotification = ClientStateConnecting
entered := make(chan struct{})
release := make(chan struct{})
l := &recordingListener{}
l.onState = func(state ClientState) {
if state != ClientStateNoNetwork {
return
}
l.mu.Lock()
l.onState = nil
l.mu.Unlock()
close(entered)
<-release
}
n.listener = l
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
n.setNetworkAvailable(false)
}()
<-entered
flipped := make(chan struct{})
go func() {
defer close(flipped)
n.setNetworkAvailable(true)
}()
select {
case <-flipped:
t.Fatal("the online transition published while the offline one was " +
"still in flight; publication is not serialized")
case <-time.After(200 * time.Millisecond):
}
close(release)
<-flipped
wg.Wait()
got, ok := l.last()
if !ok {
t.Fatal("listener never observed a state")
}
if got != ClientStateConnecting {
t.Fatalf("listener holds %v after the network came back, want Connecting; sequence: %v",
got, l.snapshot())
}
}

27
client/internal/peer/notifier_test.go
View File

@@ -6,29 +6,32 @@ import (
)
type mocListener struct {
lastState int
lastState ClientState
wg sync.WaitGroup
peersWg sync.WaitGroup
peers int
}
func (l *mocListener) OnConnected() {
l.lastState = stateConnected
l.lastState = ClientStateConnected
l.wg.Done()
}
func (l *mocListener) OnDisconnected() {
l.lastState = stateDisconnected
l.lastState = ClientStateDisconnected
l.wg.Done()
}
func (l *mocListener) OnConnecting() {
l.lastState = stateConnecting
l.lastState = ClientStateConnecting
l.wg.Done()
}
func (l *mocListener) OnDisconnecting() {
l.lastState = stateDisconnecting
l.lastState = ClientStateDisconnecting
l.wg.Done()
}
func (l *mocListener) OnStateChanged(state ClientState) {
}
func (l *mocListener) OnAddressChanged(host, addr string) {
}
@@ -57,15 +60,15 @@ func Test_notifier_serverState(t *testing.T) {
type scenario struct {
name string
expected int
expected ClientState
mgmState bool
signalState bool
}
scenarios := []scenario{
{"connected", stateConnected, true, true},
{"mgm down", stateConnecting, false, true},
{"signal down", stateConnecting, true, false},
{"disconnected", stateDisconnected, false, false},
{"connected", ClientStateConnected, true, true},
{"mgm down", ClientStateConnecting, false, true},
{"signal down", ClientStateConnecting, true, false},
{"disconnected", ClientStateDisconnected, false, false},
}
for _, tt := range scenarios {
@@ -85,7 +88,7 @@ func Test_notifier_SetListener(t *testing.T) {
listener.setPeersWaiter()
n := newNotifier()
n.lastNotification = stateConnecting
n.lastNotification = ClientStateConnecting
n.setListener(listener)
listener.wait()
listener.waitPeers()
@@ -99,7 +102,7 @@ func Test_notifier_RemoveListener(t *testing.T) {
listener.setWaiter()
listener.setPeersWaiter()
n := newNotifier()
n.lastNotification = stateConnecting
n.lastNotification = ClientStateConnecting
n.setListener(listener)
// setListener replays cached state on a goroutine; wait for both the state
// and peers callbacks to finish so we don't race on listener.peers.

6
client/internal/peer/status.go
View File

@@ -1211,6 +1211,12 @@ func (d *Status) ClientTeardown() {
d.notifyStateChange()
}
// SetNetworkAvailable records the OS-reported network availability; while
// unavailable, listeners see NoNetwork instead of Connecting.
func (d *Status) SetNetworkAvailable(available bool) {
d.notifier.setNetworkAvailable(available)
}
// SetConnectionListener set a listener to the notifier
func (d *Status) SetConnectionListener(listener Listener) {
d.notifier.setListener(listener)

21
client/internal/peer/worker_ice.go
View File

@@ -255,8 +255,8 @@ func (w *WorkerICE) connect(ctx context.Context, agent *icemaker.ThreadSafeAgent
return
}
w.log.Debugf("turn agent dial")
remoteConn, err := w.turnAgentDial(ctx, agent, remoteOfferAnswer)
w.log.Debugf("agent dial")
remoteConn, err := w.agentDial(ctx, agent, remoteOfferAnswer)
if err != nil {
w.log.Debugf("failed to dial the remote peer: %s", err)
w.closeAgent(agent, w.agentDialerCancel)
@@ -389,6 +389,17 @@ func (w *WorkerICE) injectPortForwardedCandidate(srflxCandidate ice.Candidate) {
return
}
// A forwarded candidate only makes sense for an IPv4 mapping, which
// translates a port on the gateway's address. An IPv6 pinhole translates
// nothing: it unblocks the address ICE already gathers as a host candidate,
// so there is no second address to advertise. Injecting one here would also
// paste an IPv6 address onto whichever server-reflexive candidate arrived
// first, which is usually IPv4.
if mapping.ExternalIP != nil && mapping.ExternalIP.To4() == nil {
w.log.Debugf("skipping port-forwarded candidate: %s mapping is IPv6-only", mapping.NATType)
return
}
w.muxAgent.Lock()
if w.portForwardAttempted {
w.muxAgent.Unlock()
@@ -517,8 +528,8 @@ func (w *WorkerICE) onConnectionStateChange(agent *icemaker.ThreadSafeAgent, dia
w.logSuccessfulPaths(agent)
return
case ice.ConnectionStateFailed, ice.ConnectionStateDisconnected, ice.ConnectionStateClosed:
// ice.ConnectionStateClosed happens when we recreate the agent. For the P2P to TURN switch important to
// notify the conn.onICEStateDisconnected changes to update the current used priority
// ice.ConnectionStateClosed happens when we recreate the agent. The P2P to relay switch requires
// notifying conn.onICEStateDisconnected so it can update the currently used priority.
sessionChanged := w.closeAgent(agent, dialerCancel)
@@ -532,7 +543,7 @@ func (w *WorkerICE) onConnectionStateChange(agent *icemaker.ThreadSafeAgent, dia
}
}
func (w *WorkerICE) turnAgentDial(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) {
func (w *WorkerICE) agentDial(ctx context.Context, agent *icemaker.ThreadSafeAgent, remoteOfferAnswer *OfferAnswer) (*ice.Conn, error) {
if isController(w.config) {
return agent.Dial(ctx, remoteOfferAnswer.IceCredentials.UFrag, remoteOfferAnswer.IceCredentials.Pwd)
} else {

30
client/internal/portforward/manager.go
View File

@@ -10,10 +10,8 @@ import (
"sync"
"time"
"github.com/libp2p/go-nat"
"github.com/netbirdio/go-nat"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/internal/portforward/pcp"
)
const (
@@ -168,6 +166,11 @@ func (m *Manager) setup(ctx context.Context) (nat.NAT, *Mapping, error) {
if err != nil {
return nil, nil, fmt.Errorf("create port mapping: %w", err)
}
// Only meaningful once a mapping has been attempted: that is what opens the
// pinhole and records its outcome.
logIPv6Pinhole(gateway)
return gateway, mapping, nil
}
@@ -265,7 +268,9 @@ func (m *Manager) checkHealthAndRecreate(ctx context.Context, gateway nat.NAT) b
return false
}
pcpNAT, ok := gateway.(*pcp.NAT)
// Assert on the interface, not on a concrete type: a dual-stack gateway is
// a wrapper around the IPv4 NAT, so a type assertion misses it.
checker, ok := gateway.(nat.HealthChecker)
if !ok {
return false
}
@@ -273,7 +278,7 @@ func (m *Manager) checkHealthAndRecreate(ctx context.Context, gateway nat.NAT) b
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
epoch, serverRestarted, err := pcpNAT.CheckServerHealth(ctx)
epoch, serverRestarted, err := checker.CheckServerHealth(ctx)
if err != nil {
log.Debugf("PCP health check failed: %v", err)
return false
@@ -340,3 +345,18 @@ func (m *Manager) startTearDown(ctx context.Context) {
func isPermanentLeaseRequired(err error) bool {
return err != nil && upnpErrPermanentLeaseOnly.MatchString(err.Error())
}
// logIPv6Pinhole reports the outcome of the IPv6 pinhole. Pinholes are best
// effort and never fail a mapping on their own, so this is the only way to see
// whether one was actually opened.
func logIPv6Pinhole(gateway nat.NAT) {
reporter, ok := gateway.(nat.IPv6PinholeReporter)
if !ok {
return
}
if err := reporter.IPv6PinholeError(); err != nil {
log.Warnf("IPv6 pinhole: %v", err)
return
}
log.Infof("IPv6 pinhole open")
}

408
client/internal/portforward/pcp/client.go
View File

@@ -1,408 +0,0 @@
package pcp
import (
"context"
"crypto/rand"
"errors"
"fmt"
"net"
"net/netip"
"sync"
"time"
log "github.com/sirupsen/logrus"
)
const (
defaultTimeout = 3 * time.Second
responseBufferSize = 128
// RFC 6887 Section 8.1.1 retry timing
initialRetryDelay = 3 * time.Second
maxRetryDelay = 1024 * time.Second
maxRetries = 4 // 3s + 6s + 12s + 24s = 45s total worst case
)
// Client is a PCP protocol client.
// All methods are safe for concurrent use.
type Client struct {
gateway netip.Addr
timeout time.Duration
mu sync.Mutex
// localIP caches the resolved local IP address.
localIP netip.Addr
// lastEpoch is the last observed server epoch value.
lastEpoch uint32
// epochTime tracks when lastEpoch was received for state loss detection.
epochTime time.Time
// externalIP caches the external IP from the last successful MAP response.
externalIP netip.Addr
// epochStateLost is set when epoch indicates server restart.
epochStateLost bool
}
// NewClient creates a new PCP client for the gateway at the given IP.
func NewClient(gateway net.IP) *Client {
addr, ok := netip.AddrFromSlice(gateway)
if !ok {
log.Debugf("invalid gateway IP: %v", gateway)
}
return &Client{
gateway: addr.Unmap(),
timeout: defaultTimeout,
}
}
// NewClientWithTimeout creates a new PCP client with a custom timeout.
func NewClientWithTimeout(gateway net.IP, timeout time.Duration) *Client {
addr, ok := netip.AddrFromSlice(gateway)
if !ok {
log.Debugf("invalid gateway IP: %v", gateway)
}
return &Client{
gateway: addr.Unmap(),
timeout: timeout,
}
}
// SetLocalIP sets the local IP address to use in PCP requests.
func (c *Client) SetLocalIP(ip net.IP) {
addr, ok := netip.AddrFromSlice(ip)
if !ok {
log.Debugf("invalid local IP: %v", ip)
}
c.mu.Lock()
c.localIP = addr.Unmap()
c.mu.Unlock()
}
// Gateway returns the gateway IP address.
func (c *Client) Gateway() net.IP {
return c.gateway.AsSlice()
}
// Announce sends a PCP ANNOUNCE request to discover PCP support.
// Returns the server's epoch time on success.
func (c *Client) Announce(ctx context.Context) (epoch uint32, err error) {
localIP, err := c.getLocalIP()
if err != nil {
return 0, fmt.Errorf("get local IP: %w", err)
}
req := buildAnnounceRequest(localIP)
resp, err := c.sendRequest(ctx, req)
if err != nil {
return 0, fmt.Errorf("send announce: %w", err)
}
parsed, err := parseResponse(resp)
if err != nil {
return 0, fmt.Errorf("parse announce response: %w", err)
}
if parsed.ResultCode != ResultSuccess {
return 0, fmt.Errorf("PCP ANNOUNCE failed: %s", ResultCodeString(parsed.ResultCode))
}
c.mu.Lock()
if c.updateEpochLocked(parsed.Epoch) {
log.Warnf("PCP server epoch indicates state loss - mappings may need refresh")
}
c.mu.Unlock()
return parsed.Epoch, nil
}
// AddPortMapping requests a port mapping from the PCP server.
func (c *Client) AddPortMapping(ctx context.Context, protocol string, internalPort int, lifetime time.Duration) (*MapResponse, error) {
return c.addPortMappingWithHint(ctx, protocol, internalPort, internalPort, netip.Addr{}, lifetime)
}
// AddPortMappingWithHint requests a port mapping with suggested external port and IP.
// Use lifetime <= 0 to delete a mapping.
func (c *Client) AddPortMappingWithHint(ctx context.Context, protocol string, internalPort, suggestedExtPort int, suggestedExtIP net.IP, lifetime time.Duration) (*MapResponse, error) {
var extIP netip.Addr
if suggestedExtIP != nil {
var ok bool
extIP, ok = netip.AddrFromSlice(suggestedExtIP)
if !ok {
log.Debugf("invalid suggested external IP: %v", suggestedExtIP)
}
extIP = extIP.Unmap()
}
return c.addPortMappingWithHint(ctx, protocol, internalPort, suggestedExtPort, extIP, lifetime)
}
func (c *Client) addPortMappingWithHint(ctx context.Context, protocol string, internalPort, suggestedExtPort int, suggestedExtIP netip.Addr, lifetime time.Duration) (*MapResponse, error) {
localIP, err := c.getLocalIP()
if err != nil {
return nil, fmt.Errorf("get local IP: %w", err)
}
proto, err := protocolNumber(protocol)
if err != nil {
return nil, fmt.Errorf("parse protocol: %w", err)
}
var nonce [12]byte
if _, err := rand.Read(nonce[:]); err != nil {
return nil, fmt.Errorf("generate nonce: %w", err)
}
// Convert lifetime to seconds. Lifetime 0 means delete, so only apply
// default for positive durations that round to 0 seconds.
var lifetimeSec uint32
if lifetime > 0 {
lifetimeSec = uint32(lifetime.Seconds())
if lifetimeSec == 0 {
lifetimeSec = DefaultLifetime
}
}
req := buildMapRequest(localIP, nonce, proto, uint16(internalPort), uint16(suggestedExtPort), suggestedExtIP, lifetimeSec)
resp, err := c.sendRequest(ctx, req)
if err != nil {
return nil, fmt.Errorf("send map request: %w", err)
}
mapResp, err := parseMapResponse(resp)
if err != nil {
return nil, fmt.Errorf("parse map response: %w", err)
}
if mapResp.Nonce != nonce {
return nil, fmt.Errorf("nonce mismatch in response")
}
if mapResp.Protocol != proto {
return nil, fmt.Errorf("protocol mismatch: requested %d, got %d", proto, mapResp.Protocol)
}
if mapResp.InternalPort != uint16(internalPort) {
return nil, fmt.Errorf("internal port mismatch: requested %d, got %d", internalPort, mapResp.InternalPort)
}
if mapResp.ResultCode != ResultSuccess {
return nil, &Error{
Code: mapResp.ResultCode,
Message: ResultCodeString(mapResp.ResultCode),
}
}
c.mu.Lock()
if c.updateEpochLocked(mapResp.Epoch) {
log.Warnf("PCP server epoch indicates state loss - mappings may need refresh")
}
c.cacheExternalIPLocked(mapResp.ExternalIP)
c.mu.Unlock()
return mapResp, nil
}
// DeletePortMapping removes a port mapping by requesting zero lifetime.
func (c *Client) DeletePortMapping(ctx context.Context, protocol string, internalPort int) error {
if _, err := c.addPortMappingWithHint(ctx, protocol, internalPort, 0, netip.Addr{}, 0); err != nil {
var pcpErr *Error
if errors.As(err, &pcpErr) && pcpErr.Code == ResultNotAuthorized {
return nil
}
return fmt.Errorf("delete mapping: %w", err)
}
return nil
}
// GetExternalAddress returns the external IP address.
// First checks for a cached value from previous MAP responses.
// If not cached, creates a short-lived mapping to discover the external IP.
func (c *Client) GetExternalAddress(ctx context.Context) (net.IP, error) {
c.mu.Lock()
if c.externalIP.IsValid() {
ip := c.externalIP.AsSlice()
c.mu.Unlock()
return ip, nil
}
c.mu.Unlock()
// Use an ephemeral port in the dynamic range (49152-65535).
// Port 0 is not valid with UDP/TCP protocols per RFC 6887.
ephemeralPort := 49152 + int(uint16(time.Now().UnixNano()))%(65535-49152)
// Use minimal lifetime (1 second) for discovery.
resp, err := c.AddPortMapping(ctx, "udp", ephemeralPort, time.Second)
if err != nil {
return nil, fmt.Errorf("create temporary mapping: %w", err)
}
if err := c.DeletePortMapping(ctx, "udp", ephemeralPort); err != nil {
log.Debugf("cleanup temporary PCP mapping: %v", err)
}
return resp.ExternalIP.AsSlice(), nil
}
// LastEpoch returns the last observed server epoch value.
// A decrease in epoch indicates the server may have restarted and mappings may be lost.
func (c *Client) LastEpoch() uint32 {
c.mu.Lock()
defer c.mu.Unlock()
return c.lastEpoch
}
// EpochStateLost returns true if epoch state loss was detected and clears the flag.
func (c *Client) EpochStateLost() bool {
c.mu.Lock()
defer c.mu.Unlock()
lost := c.epochStateLost
c.epochStateLost = false
return lost
}
// updateEpoch updates the epoch tracking and detects potential state loss.
// Returns true if state loss was detected (server likely restarted).
// Caller must hold c.mu.
func (c *Client) updateEpochLocked(newEpoch uint32) bool {
now := time.Now()
stateLost := false
// RFC 6887 Section 8.5: Detect invalid epoch indicating server state loss.
// client_delta = time since last response
// server_delta = epoch change since last response
// Invalid if: client_delta+2 < server_delta - server_delta/16
// OR: server_delta+2 < client_delta - client_delta/16
// The +2 handles quantization, /16 (6.25%) handles clock drift.
if !c.epochTime.IsZero() && c.lastEpoch > 0 {
clientDelta := uint32(now.Sub(c.epochTime).Seconds())
serverDelta := newEpoch - c.lastEpoch
// Check for epoch going backwards or jumping unexpectedly.
// Subtraction is safe: serverDelta/16 is always <= serverDelta.
if clientDelta+2 < serverDelta-(serverDelta/16) ||
serverDelta+2 < clientDelta-(clientDelta/16) {
stateLost = true
c.epochStateLost = true
}
}
c.lastEpoch = newEpoch
c.epochTime = now
return stateLost
}
// cacheExternalIP stores the external IP from a successful MAP response.
// Caller must hold c.mu.
func (c *Client) cacheExternalIPLocked(ip netip.Addr) {
if ip.IsValid() && !ip.IsUnspecified() {
c.externalIP = ip
}
}
// sendRequest sends a PCP request with retries per RFC 6887 Section 8.1.1.
func (c *Client) sendRequest(ctx context.Context, req []byte) ([]byte, error) {
addr := &net.UDPAddr{IP: c.gateway.AsSlice(), Port: Port}
var lastErr error
delay := initialRetryDelay
for range maxRetries {
resp, err := c.sendOnce(ctx, addr, req)
if err == nil {
return resp, nil
}
lastErr = err
if ctx.Err() != nil {
return nil, ctx.Err()
}
// RFC 6887 Section 8.1.1: RT = (1 + RAND) * MIN(2 * RTprev, MRT)
// RAND is random between -0.1 and +0.1
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(retryDelayWithJitter(delay)):
}
delay = min(delay*2, maxRetryDelay)
}
return nil, fmt.Errorf("PCP request failed after %d retries: %w", maxRetries, lastErr)
}
// retryDelayWithJitter applies RFC 6887 jitter: multiply by (1 + RAND) where RAND is [-0.1, +0.1].
func retryDelayWithJitter(d time.Duration) time.Duration {
var b [1]byte
_, _ = rand.Read(b[:])
// Convert byte to range [-0.1, +0.1]: (b/255 * 0.2) - 0.1
jitter := (float64(b[0])/255.0)*0.2 - 0.1
return time.Duration(float64(d) * (1 + jitter))
}
func (c *Client) sendOnce(ctx context.Context, addr *net.UDPAddr, req []byte) ([]byte, error) {
// Use ListenUDP instead of DialUDP to validate response source address per RFC 6887 §8.3.
conn, err := net.ListenUDP("udp", nil)
if err != nil {
return nil, fmt.Errorf("listen: %w", err)
}
defer func() {
if err := conn.Close(); err != nil {
log.Debugf("close UDP connection: %v", err)
}
}()
timeout := c.timeout
if deadline, ok := ctx.Deadline(); ok {
if remaining := time.Until(deadline); remaining < timeout {
timeout = remaining
}
}
if err := conn.SetDeadline(time.Now().Add(timeout)); err != nil {
return nil, fmt.Errorf("set deadline: %w", err)
}
if _, err := conn.WriteToUDP(req, addr); err != nil {
return nil, fmt.Errorf("write: %w", err)
}
resp := make([]byte, responseBufferSize)
n, from, err := conn.ReadFromUDP(resp)
if err != nil {
return nil, fmt.Errorf("read: %w", err)
}
// RFC 6887 §8.3: Validate response came from expected PCP server.
if !from.IP.Equal(addr.IP) {
return nil, fmt.Errorf("response from unexpected source %s (expected %s)", from.IP, addr.IP)
}
return resp[:n], nil
}
func (c *Client) getLocalIP() (netip.Addr, error) {
c.mu.Lock()
defer c.mu.Unlock()
if !c.localIP.IsValid() {
return netip.Addr{}, fmt.Errorf("local IP not set for gateway %s", c.gateway)
}
return c.localIP, nil
}
func protocolNumber(protocol string) (uint8, error) {
switch protocol {
case "udp", "UDP":
return ProtoUDP, nil
case "tcp", "TCP":
return ProtoTCP, nil
default:
return 0, fmt.Errorf("unsupported protocol: %s", protocol)
}
}
// Error represents a PCP error response.
type Error struct {
Code uint8
Message string
}
func (e *Error) Error() string {
return fmt.Sprintf("PCP error: %s (%d)", e.Message, e.Code)
}

187
client/internal/portforward/pcp/client_test.go
View File

@@ -1,187 +0,0 @@
package pcp
import (
"context"
"net"
"net/netip"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAddrConversion(t *testing.T) {
tests := []struct {
name string
addr netip.Addr
}{
{"IPv4", netip.MustParseAddr("192.168.1.100")},
{"IPv4 loopback", netip.MustParseAddr("127.0.0.1")},
{"IPv6", netip.MustParseAddr("2001:db8::1")},
{"IPv6 loopback", netip.MustParseAddr("::1")},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b16 := addrTo16(tt.addr)
recovered := addrFrom16(b16)
assert.Equal(t, tt.addr, recovered, "address should round-trip")
})
}
}
func TestBuildAnnounceRequest(t *testing.T) {
clientIP := netip.MustParseAddr("192.168.1.100")
req := buildAnnounceRequest(clientIP)
require.Len(t, req, headerSize)
assert.Equal(t, byte(Version), req[0], "version")
assert.Equal(t, byte(OpAnnounce), req[1], "opcode")
// Check client IP is properly encoded as IPv4-mapped IPv6
assert.Equal(t, byte(0xff), req[18], "IPv4-mapped prefix byte 10")
assert.Equal(t, byte(0xff), req[19], "IPv4-mapped prefix byte 11")
assert.Equal(t, byte(192), req[20], "IP octet 1")
assert.Equal(t, byte(168), req[21], "IP octet 2")
assert.Equal(t, byte(1), req[22], "IP octet 3")
assert.Equal(t, byte(100), req[23], "IP octet 4")
}
func TestBuildMapRequest(t *testing.T) {
clientIP := netip.MustParseAddr("192.168.1.100")
nonce := [12]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}
req := buildMapRequest(clientIP, nonce, ProtoUDP, 51820, 51820, netip.Addr{}, 3600)
require.Len(t, req, mapRequestSize)
assert.Equal(t, byte(Version), req[0], "version")
assert.Equal(t, byte(OpMap), req[1], "opcode")
// Lifetime at bytes 4-7
assert.Equal(t, uint32(3600), (uint32(req[4])<<24)|(uint32(req[5])<<16)|(uint32(req[6])<<8)|uint32(req[7]), "lifetime")
// Nonce at bytes 24-35
assert.Equal(t, nonce[:], req[24:36], "nonce")
// Protocol at byte 36
assert.Equal(t, byte(ProtoUDP), req[36], "protocol")
// Internal port at bytes 40-41
assert.Equal(t, uint16(51820), (uint16(req[40])<<8)|uint16(req[41]), "internal port")
// External port at bytes 42-43
assert.Equal(t, uint16(51820), (uint16(req[42])<<8)|uint16(req[43]), "external port")
}
func TestParseResponse(t *testing.T) {
// Construct a valid ANNOUNCE response
resp := make([]byte, headerSize)
resp[0] = Version
resp[1] = OpAnnounce | OpReply
// Result code = 0 (success)
// Lifetime = 0
// Epoch = 12345
resp[8] = 0
resp[9] = 0
resp[10] = 0x30
resp[11] = 0x39
parsed, err := parseResponse(resp)
require.NoError(t, err)
assert.Equal(t, uint8(Version), parsed.Version)
assert.Equal(t, uint8(OpAnnounce|OpReply), parsed.Opcode)
assert.Equal(t, uint8(ResultSuccess), parsed.ResultCode)
assert.Equal(t, uint32(12345), parsed.Epoch)
}
func TestParseResponseErrors(t *testing.T) {
t.Run("too short", func(t *testing.T) {
_, err := parseResponse([]byte{1, 2, 3})
assert.Error(t, err)
})
t.Run("wrong version", func(t *testing.T) {
resp := make([]byte, headerSize)
resp[0] = 1 // Wrong version
resp[1] = OpReply
_, err := parseResponse(resp)
assert.Error(t, err)
})
t.Run("missing reply bit", func(t *testing.T) {
resp := make([]byte, headerSize)
resp[0] = Version
resp[1] = OpAnnounce // Missing OpReply bit
_, err := parseResponse(resp)
assert.Error(t, err)
})
}
func TestResultCodeString(t *testing.T) {
assert.Equal(t, "SUCCESS", ResultCodeString(ResultSuccess))
assert.Equal(t, "NOT_AUTHORIZED", ResultCodeString(ResultNotAuthorized))
assert.Equal(t, "ADDRESS_MISMATCH", ResultCodeString(ResultAddressMismatch))
assert.Contains(t, ResultCodeString(255), "UNKNOWN")
}
func TestProtocolNumber(t *testing.T) {
proto, err := protocolNumber("udp")
require.NoError(t, err)
assert.Equal(t, uint8(ProtoUDP), proto)
proto, err = protocolNumber("tcp")
require.NoError(t, err)
assert.Equal(t, uint8(ProtoTCP), proto)
proto, err = protocolNumber("UDP")
require.NoError(t, err)
assert.Equal(t, uint8(ProtoUDP), proto)
_, err = protocolNumber("icmp")
assert.Error(t, err)
}
func TestClientCreation(t *testing.T) {
gateway := netip.MustParseAddr("192.168.1.1").AsSlice()
client := NewClient(gateway)
assert.Equal(t, net.IP(gateway), client.Gateway())
assert.Equal(t, defaultTimeout, client.timeout)
clientWithTimeout := NewClientWithTimeout(gateway, 5*time.Second)
assert.Equal(t, 5*time.Second, clientWithTimeout.timeout)
}
func TestNATType(t *testing.T) {
n := NewNAT(netip.MustParseAddr("192.168.1.1").AsSlice(), netip.MustParseAddr("192.168.1.100").AsSlice())
assert.Equal(t, "PCP", n.Type())
}
// Integration test - skipped unless PCP_TEST_GATEWAY env is set
func TestClientIntegration(t *testing.T) {
t.Skip("Integration test - run manually with PCP_TEST_GATEWAY=<gateway-ip>")
gateway := netip.MustParseAddr("10.0.1.1").AsSlice() // Change to your test gateway
localIP := netip.MustParseAddr("10.0.1.100").AsSlice() // Change to your local IP
client := NewClient(gateway)
client.SetLocalIP(localIP)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
// Test ANNOUNCE
epoch, err := client.Announce(ctx)
require.NoError(t, err)
t.Logf("Server epoch: %d", epoch)
// Test MAP
resp, err := client.AddPortMapping(ctx, "udp", 51820, 1*time.Hour)
require.NoError(t, err)
t.Logf("Mapping: internal=%d external=%d externalIP=%s",
resp.InternalPort, resp.ExternalPort, resp.ExternalIP)
// Cleanup
err = client.DeletePortMapping(ctx, "udp", 51820)
require.NoError(t, err)
}

222
client/internal/portforward/pcp/nat.go
View File

@@ -1,222 +0,0 @@
package pcp
import (
"context"
"fmt"
"net"
"net/netip"
"runtime"
"sync"
"time"
log "github.com/sirupsen/logrus"
"github.com/libp2p/go-nat"
"github.com/libp2p/go-netroute"
)
var _ nat.NAT = (*NAT)(nil)
// NAT implements the go-nat NAT interface using PCP.
// Supports dual-stack (IPv4 and IPv6) when available.
// All methods are safe for concurrent use.
//
// TODO: IPv6 pinholes use the local IPv6 address. If the address changes
// (e.g., due to SLAAC rotation or network change), the pinhole becomes stale
// and needs to be recreated with the new address.
type NAT struct {
client *Client
mu sync.RWMutex
// client6 is the IPv6 PCP client, nil if IPv6 is unavailable.
client6 *Client
// localIP6 caches the local IPv6 address used for PCP requests.
localIP6 netip.Addr
}
// NewNAT creates a new NAT instance backed by PCP.
func NewNAT(gateway, localIP net.IP) *NAT {
client := NewClient(gateway)
client.SetLocalIP(localIP)
return &NAT{
client: client,
}
}
// Type returns "PCP" as the NAT type.
func (n *NAT) Type() string {
return "PCP"
}
// GetDeviceAddress returns the gateway IP address.
func (n *NAT) GetDeviceAddress() (net.IP, error) {
return n.client.Gateway(), nil
}
// GetExternalAddress returns the external IP address.
func (n *NAT) GetExternalAddress() (net.IP, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
return n.client.GetExternalAddress(ctx)
}
// GetInternalAddress returns the local IP address used to communicate with the gateway.
func (n *NAT) GetInternalAddress() (net.IP, error) {
addr, err := n.client.getLocalIP()
if err != nil {
return nil, err
}
return addr.AsSlice(), nil
}
// AddPortMapping creates a port mapping on both IPv4 and IPv6 (if available).
func (n *NAT) AddPortMapping(ctx context.Context, protocol string, internalPort int, _ string, timeout time.Duration) (int, error) {
resp, err := n.client.AddPortMapping(ctx, protocol, internalPort, timeout)
if err != nil {
return 0, fmt.Errorf("add mapping: %w", err)
}
n.mu.RLock()
client6 := n.client6
localIP6 := n.localIP6
n.mu.RUnlock()
if client6 == nil {
return int(resp.ExternalPort), nil
}
if _, err := client6.AddPortMapping(ctx, protocol, internalPort, timeout); err != nil {
log.Warnf("IPv6 PCP mapping failed (continuing with IPv4): %v", err)
return int(resp.ExternalPort), nil
}
log.Infof("created IPv6 PCP pinhole: %s:%d", localIP6, internalPort)
return int(resp.ExternalPort), nil
}
// DeletePortMapping removes a port mapping from both IPv4 and IPv6.
func (n *NAT) DeletePortMapping(ctx context.Context, protocol string, internalPort int) error {
err := n.client.DeletePortMapping(ctx, protocol, internalPort)
n.mu.RLock()
client6 := n.client6
n.mu.RUnlock()
if client6 != nil {
if err6 := client6.DeletePortMapping(ctx, protocol, internalPort); err6 != nil {
log.Warnf("IPv6 PCP delete mapping failed: %v", err6)
}
}
if err != nil {
return fmt.Errorf("delete mapping: %w", err)
}
return nil
}
// CheckServerHealth sends an ANNOUNCE to verify the server is still responsive.
// Returns the current epoch and whether the server may have restarted (epoch state loss detected).
func (n *NAT) CheckServerHealth(ctx context.Context) (epoch uint32, serverRestarted bool, err error) {
epoch, err = n.client.Announce(ctx)
if err != nil {
return 0, false, fmt.Errorf("announce: %w", err)
}
return epoch, n.client.EpochStateLost(), nil
}
// DiscoverPCP attempts to discover a PCP-capable gateway.
// Returns a NAT interface if PCP is supported, or an error otherwise.
// Discovers both IPv4 and IPv6 gateways when available.
func DiscoverPCP(ctx context.Context) (nat.NAT, error) {
gateway, localIP, err := getDefaultGateway()
if err != nil {
return nil, fmt.Errorf("get default gateway: %w", err)
}
client := NewClient(gateway)
client.SetLocalIP(localIP)
if _, err := client.Announce(ctx); err != nil {
return nil, fmt.Errorf("PCP announce: %w", err)
}
result := &NAT{client: client}
discoverIPv6(ctx, result)
return result, nil
}
func discoverIPv6(ctx context.Context, result *NAT) {
gateway6, localIP6, err := getDefaultGateway6()
if err != nil {
log.Debugf("IPv6 gateway discovery failed: %v", err)
return
}
client6 := NewClient(gateway6)
client6.SetLocalIP(localIP6)
if _, err := client6.Announce(ctx); err != nil {
log.Debugf("PCP IPv6 announce failed: %v", err)
return
}
addr, ok := netip.AddrFromSlice(localIP6)
if !ok {
log.Debugf("invalid IPv6 local IP: %v", localIP6)
return
}
result.mu.Lock()
result.client6 = client6
result.localIP6 = addr
result.mu.Unlock()
log.Debugf("PCP IPv6 gateway discovered: %s (local: %s)", gateway6, localIP6)
}
// getDefaultGateway returns the default IPv4 gateway and local IP using the system routing table.
func getDefaultGateway() (gateway net.IP, localIP net.IP, err error) {
router, err := netroute.New()
if err != nil {
return nil, nil, err
}
dst := net.IPv4zero
if runtime.GOOS == "linux" || runtime.GOOS == "android" {
// go-netroute v0.4.0 rejects unspecified destinations client-side on Linux/Android.
// TODO: on android/ios, use platform APIs (ConnectivityManager.getLinkProperties /
// NWPathMonitor) when netlink-based lookup is restricted or unavailable.
dst = net.IPv4(0, 0, 0, 1)
}
_, gateway, localIP, err = router.Route(dst)
if err != nil {
return nil, nil, err
}
if gateway == nil {
return nil, nil, nat.ErrNoNATFound
}
return gateway, localIP, nil
}
// getDefaultGateway6 returns the default IPv6 gateway IP address using the system routing table.
func getDefaultGateway6() (gateway net.IP, localIP net.IP, err error) {
router, err := netroute.New()
if err != nil {
return nil, nil, err
}
dst := net.IPv6zero
if runtime.GOOS == "linux" || runtime.GOOS == "android" {
// ::2
dst = net.IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}
}
_, gateway, localIP, err = router.Route(dst)
if err != nil {
return nil, nil, err
}
if gateway == nil {
return nil, nil, nat.ErrNoNATFound
}
return gateway, localIP, nil
}

225
client/internal/portforward/pcp/protocol.go
View File

@@ -1,225 +0,0 @@
// Package pcp implements the Port Control Protocol (RFC 6887).
//
// # Implemented Features
//
// - ANNOUNCE opcode: Discovers PCP server support
// - MAP opcode: Creates/deletes port mappings (IPv4 NAT) and firewall pinholes (IPv6)
// - Dual-stack: Simultaneous IPv4 and IPv6 support via separate clients
// - Nonce validation: Prevents response spoofing
// - Epoch tracking: Detects server restarts per Section 8.5
// - RFC-compliant retry timing: 3s initial, exponential backoff to 1024s max (Section 8.1.1)
//
// # Not Implemented
//
// - PEER opcode: For outbound peer connections (not needed for inbound NAT traversal)
// - THIRD_PARTY option: For managing mappings on behalf of other devices
// - PREFER_FAILURE option: Requires exact external port or fail (IPv4 NAT only, not needed for IPv6 pinholing)
// - FILTER option: To restrict remote peer addresses
//
// These optional features are omitted because the primary use case is simple
// port forwarding for WireGuard, which only requires MAP with default behavior.
package pcp
import (
"encoding/binary"
"fmt"
"net/netip"
)
const (
// Version is the PCP protocol version (RFC 6887).
Version = 2
// Port is the standard PCP server port.
Port = 5351
// DefaultLifetime is the default requested mapping lifetime in seconds.
DefaultLifetime = 7200 // 2 hours
// Header sizes
headerSize = 24
mapPayloadSize = 36
mapRequestSize = headerSize + mapPayloadSize // 60 bytes
)
// Opcodes
const (
OpAnnounce = 0
OpMap = 1
OpPeer = 2
OpReply = 0x80 // OR'd with opcode in responses
)
// Protocol numbers for MAP requests
const (
ProtoUDP = 17
ProtoTCP = 6
)
// Result codes (RFC 6887 Section 7.4)
const (
ResultSuccess = 0
ResultUnsuppVersion = 1
ResultNotAuthorized = 2
ResultMalformedRequest = 3
ResultUnsuppOpcode = 4
ResultUnsuppOption = 5
ResultMalformedOption = 6
ResultNetworkFailure = 7
ResultNoResources = 8
ResultUnsuppProtocol = 9
ResultUserExQuota = 10
ResultCannotProvideExt = 11
ResultAddressMismatch = 12
ResultExcessiveRemotePeers = 13
)
// ResultCodeString returns a human-readable string for a result code.
func ResultCodeString(code uint8) string {
switch code {
case ResultSuccess:
return "SUCCESS"
case ResultUnsuppVersion:
return "UNSUPP_VERSION"
case ResultNotAuthorized:
return "NOT_AUTHORIZED"
case ResultMalformedRequest:
return "MALFORMED_REQUEST"
case ResultUnsuppOpcode:
return "UNSUPP_OPCODE"
case ResultUnsuppOption:
return "UNSUPP_OPTION"
case ResultMalformedOption:
return "MALFORMED_OPTION"
case ResultNetworkFailure:
return "NETWORK_FAILURE"
case ResultNoResources:
return "NO_RESOURCES"
case ResultUnsuppProtocol:
return "UNSUPP_PROTOCOL"
case ResultUserExQuota:
return "USER_EX_QUOTA"
case ResultCannotProvideExt:
return "CANNOT_PROVIDE_EXTERNAL"
case ResultAddressMismatch:
return "ADDRESS_MISMATCH"
case ResultExcessiveRemotePeers:
return "EXCESSIVE_REMOTE_PEERS"
default:
return fmt.Sprintf("UNKNOWN(%d)", code)
}
}
// Response represents a parsed PCP response header.
type Response struct {
Version uint8
Opcode uint8
ResultCode uint8
Lifetime uint32
Epoch uint32
}
// MapResponse contains the full response to a MAP request.
type MapResponse struct {
Response
Nonce [12]byte
Protocol uint8
InternalPort uint16
ExternalPort uint16
ExternalIP netip.Addr
}
// addrTo16 converts an address to its 16-byte IPv4-mapped IPv6 representation.
func addrTo16(addr netip.Addr) [16]byte {
if addr.Is4() {
return netip.AddrFrom4(addr.As4()).As16()
}
return addr.As16()
}
// addrFrom16 extracts an address from a 16-byte representation, unmapping IPv4.
func addrFrom16(b [16]byte) netip.Addr {
return netip.AddrFrom16(b).Unmap()
}
// buildAnnounceRequest creates a PCP ANNOUNCE request packet.
func buildAnnounceRequest(clientIP netip.Addr) []byte {
req := make([]byte, headerSize)
req[0] = Version
req[1] = OpAnnounce
mapped := addrTo16(clientIP)
copy(req[8:24], mapped[:])
return req
}
// buildMapRequest creates a PCP MAP request packet.
func buildMapRequest(clientIP netip.Addr, nonce [12]byte, protocol uint8, internalPort, suggestedExtPort uint16, suggestedExtIP netip.Addr, lifetime uint32) []byte {
req := make([]byte, mapRequestSize)
// Header
req[0] = Version
req[1] = OpMap
binary.BigEndian.PutUint32(req[4:8], lifetime)
mapped := addrTo16(clientIP)
copy(req[8:24], mapped[:])
// MAP payload
copy(req[24:36], nonce[:])
req[36] = protocol
binary.BigEndian.PutUint16(req[40:42], internalPort)
binary.BigEndian.PutUint16(req[42:44], suggestedExtPort)
if suggestedExtIP.IsValid() {
extMapped := addrTo16(suggestedExtIP)
copy(req[44:60], extMapped[:])
}
return req
}
// parseResponse parses the common PCP response header.
func parseResponse(data []byte) (*Response, error) {
if len(data) < headerSize {
return nil, fmt.Errorf("response too short: %d bytes", len(data))
}
resp := &Response{
Version: data[0],
Opcode: data[1],
ResultCode: data[3], // Byte 2 is reserved, byte 3 is result code (RFC 6887 §7.2)
Lifetime: binary.BigEndian.Uint32(data[4:8]),
Epoch: binary.BigEndian.Uint32(data[8:12]),
}
if resp.Version != Version {
return nil, fmt.Errorf("unsupported PCP version: %d", resp.Version)
}
if resp.Opcode&OpReply == 0 {
return nil, fmt.Errorf("response missing reply bit: opcode=0x%02x", resp.Opcode)
}
return resp, nil
}
// parseMapResponse parses a complete MAP response.
func parseMapResponse(data []byte) (*MapResponse, error) {
if len(data) < mapRequestSize {
return nil, fmt.Errorf("MAP response too short: %d bytes", len(data))
}
resp, err := parseResponse(data)
if err != nil {
return nil, fmt.Errorf("parse header: %w", err)
}
mapResp := &MapResponse{
Response: *resp,
Protocol: data[36],
InternalPort: binary.BigEndian.Uint16(data[40:42]),
ExternalPort: binary.BigEndian.Uint16(data[42:44]),
ExternalIP: addrFrom16([16]byte(data[44:60])),
}
copy(mapResp.Nonce[:], data[24:36])
return mapResp, nil
}

116
client/internal/portforward/pinhole_test.go Normal file
View File

@@ -0,0 +1,116 @@
//go:build !js
package portforward
import (
"context"
"errors"
"strings"
"testing"
"github.com/netbirdio/go-nat"
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// mockPinholeNAT is a gateway that also reports an IPv6 pinhole outcome, the
// shape a dual-stack gateway has.
type mockPinholeNAT struct {
*mockNAT
pinholeErr error
}
func (m *mockPinholeNAT) IPv6PinholeError() error {
return m.pinholeErr
}
func TestSetupLogsPinholeOutcome(t *testing.T) {
pinholeErr := errors.New("pcp ipv6: NOT_AUTHORIZED")
tests := []struct {
name string
pinholeErr error
mappingErr error
wantLevel log.Level
wantText string
}{
{
name: "an open pinhole is reported",
wantLevel: log.InfoLevel,
wantText: "IPv6 pinhole open",
},
{
name: "a failed pinhole is reported without failing the mapping",
// The IPv4 mapping is what the caller asked for, so the pinhole
// failure surfaces only in the log.
pinholeErr: pinholeErr,
wantLevel: log.WarnLevel,
wantText: pinholeErr.Error(),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gateway := &mockPinholeNAT{mockNAT: newMockNAT(), pinholeErr: tt.pinholeErr}
hook := stubGatewayDiscovery(t, gateway)
m := NewManager()
m.wgPort = 51820
_, mapping, err := m.setup(context.Background())
require.NoError(t, err)
require.NotNil(t, mapping)
entry := findEntry(hook, tt.wantText)
require.NotNil(t, entry, "no log entry mentioning %q", tt.wantText)
assert.Equal(t, tt.wantLevel, entry.Level)
})
}
t.Run("a failed mapping reports no pinhole outcome", func(t *testing.T) {
// Nothing opened the pinhole, so whatever it currently reports says
// nothing about this attempt.
gateway := &mockPinholeNAT{mockNAT: newMockNAT()}
gateway.addMappingErr = errors.New("gateway refused")
hook := stubGatewayDiscovery(t, gateway)
m := NewManager()
m.wgPort = 51820
_, _, err := m.setup(context.Background())
require.Error(t, err)
assert.Nil(t, findEntry(hook, "IPv6 pinhole"))
})
}
// stubGatewayDiscovery makes discovery return gateway and captures log output.
func stubGatewayDiscovery(t *testing.T, gateway nat.NAT) *test.Hook {
t.Helper()
orig := discoverGateway
discoverGateway = func(context.Context) (nat.NAT, error) { return gateway, nil }
t.Cleanup(func() { discoverGateway = orig })
hook := test.NewGlobal()
origLevel := log.GetLevel()
log.SetLevel(log.DebugLevel)
t.Cleanup(func() {
hook.Reset()
log.SetLevel(origLevel)
})
return hook
}
func findEntry(hook *test.Hook, substr string) *log.Entry {
for _, entry := range hook.AllEntries() {
if strings.Contains(entry.Message, substr) {
return entry
}
}
return nil
}

89
client/internal/portforward/state.go
View File

@@ -4,27 +4,94 @@ package portforward
import (
"context"
"errors"
"fmt"
"time"
"github.com/libp2p/go-nat"
"github.com/netbirdio/go-nat"
"github.com/netbirdio/go-nat/pcp"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/internal/portforward/pcp"
)
// discoverGateway is the function used for NAT gateway discovery.
// It can be replaced in tests to avoid real network operations.
// Tries PCP first, then falls back to NAT-PMP/UPnP.
var discoverGateway = defaultDiscoverGateway
func defaultDiscoverGateway(ctx context.Context) (nat.NAT, error) {
pcpGateway, err := pcp.DiscoverPCP(ctx)
if err == nil {
return pcpGateway, nil
}
log.Debugf("PCP discovery failed: %v, trying NAT-PMP/UPnP", err)
// pinholeDiscoveryTimeout is the slice of the discovery budget held back for
// the IPv6 pinhole probe.
//
// Sizing it is coarser than it looks: PCP retransmits on a 3s socket timeout
// and a 3s first backoff, so a second attempt needs about 9s. Anything from
// roughly 1s to 8s therefore buys exactly one attempt, and this only sets how
// long that attempt waits. A PCP server sits on the local link and answers in
// milliseconds, so 3s is margin rather than need, and the rest is left to
// gateway discovery, whose multicast SSDP search alone takes 5s. A probe lost
// to a dropped packet is retried by the next discovery round.
//
// It is a variable so tests can shorten it.
var pinholeDiscoveryTimeout = 3 * time.Second
return nat.DiscoverGateway(ctx)
// Discovery entry points, as variables so tests can drive the fallback without
// touching the network.
var (
discoverNATGateway = nat.DiscoverGateway
discoverPCPPinhole = func(ctx context.Context) (nat.NAT, error) {
pinhole, err := pcp.DiscoverPCP(ctx)
if err != nil {
return nil, err
}
return pinhole, nil
}
)
// defaultDiscoverGateway finds a gateway that can make the WireGuard port
// reachable. DiscoverGateway prefers PCP for IPv4, races UPnP and NAT-PMP
// behind it, and attaches an IPv6 pinhole independently of which IPv4 protocol
// wins.
//
// It reports no gateway on a network offering only IPv6, having no IPv4 mapping
// to attach a pinhole to. Such a network still needs one: there is no
// translation to traverse, but the router drops inbound IPv6 until something
// opens it. Fall back to PCP alone, which yields a gateway holding just the
// pinhole.
func defaultDiscoverGateway(ctx context.Context) (nat.NAT, error) {
gatewayCtx, cancel := reserveForPinhole(ctx)
defer cancel()
gateway, err := discoverNATGateway(gatewayCtx)
if err == nil {
return gateway, nil
}
if !errors.Is(err, nat.ErrNoNATFound) {
return nil, err
}
pinhole, pinholeErr := discoverPCPPinhole(ctx)
if pinholeErr != nil {
log.Debugf("no IPv6 pinhole after %v: %v", err, pinholeErr)
return nil, err
}
log.Infof("no IPv4 gateway, continuing with an IPv6 pinhole only")
return pinhole, nil
}
// reserveForPinhole shortens ctx so that a pinhole probe still has time to run
// afterwards. Finding nothing takes gateway discovery everything it is given,
// so on the unshortened context the probe would start already expired. A budget
// too small to divide is left to gateway discovery, which is the likelier win.
func reserveForPinhole(ctx context.Context) (context.Context, context.CancelFunc) {
deadline, ok := ctx.Deadline()
if !ok {
return context.WithCancel(ctx)
}
remaining := time.Until(deadline)
if remaining <= pinholeDiscoveryTimeout {
return context.WithCancel(ctx)
}
return context.WithTimeout(ctx, remaining-pinholeDiscoveryTimeout)
}
// State is persisted only for crash recovery cleanup

140
client/internal/portforward/state_test.go Normal file
View File

@@ -0,0 +1,140 @@
//go:build !js
package portforward
import (
"context"
"errors"
"testing"
"time"
"github.com/netbirdio/go-nat"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// stubDiscovery replaces both discovery entry points for the duration of a
// test. gatewayDelay simulates gateway discovery spending everything it is
// given before reporting that it found nothing.
func stubDiscovery(t *testing.T, gateway nat.NAT, gatewayErr error, gatewayDelay time.Duration, pinhole nat.NAT, pinholeErr error) {
t.Helper()
origGateway, origPinhole := discoverNATGateway, discoverPCPPinhole
discoverNATGateway = func(ctx context.Context) (nat.NAT, error) {
if gatewayDelay > 0 {
select {
case <-time.After(gatewayDelay):
case <-ctx.Done():
}
}
return gateway, gatewayErr
}
discoverPCPPinhole = func(ctx context.Context) (nat.NAT, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
return pinhole, pinholeErr
}
t.Cleanup(func() { discoverNATGateway, discoverPCPPinhole = origGateway, origPinhole })
}
func TestDefaultDiscoverGateway(t *testing.T) {
ipv4Gateway := &mockNAT{natType: "PCP+PCPv6"}
ipv6Pinhole := &mockNAT{natType: "PCP"}
otherErr := errors.New("routing table unavailable")
t.Run("an IPv4 gateway is used as is", func(t *testing.T) {
stubDiscovery(t, ipv4Gateway, nil, 0, ipv6Pinhole, nil)
got, err := defaultDiscoverGateway(context.Background())
require.NoError(t, err)
assert.Same(t, ipv4Gateway, got)
})
t.Run("no IPv4 gateway still opens an IPv6 pinhole", func(t *testing.T) {
stubDiscovery(t, nil, nat.ErrNoNATFound, 0, ipv6Pinhole, nil)
got, err := defaultDiscoverGateway(context.Background())
require.NoError(t, err)
assert.Same(t, ipv6Pinhole, got)
})
t.Run("no gateway and no pinhole reports the original failure", func(t *testing.T) {
stubDiscovery(t, nil, nat.ErrNoNATFound, 0, nil, errors.New("no IPv6 route"))
got, err := defaultDiscoverGateway(context.Background())
assert.Nil(t, got)
assert.ErrorIs(t, err, nat.ErrNoNATFound, "the pinhole failure must not mask why no gateway was found")
})
t.Run("a failure other than no-gateway is reported as is", func(t *testing.T) {
stubDiscovery(t, nil, otherErr, 0, ipv6Pinhole, nil)
got, err := defaultDiscoverGateway(context.Background())
assert.Nil(t, got)
assert.ErrorIs(t, err, otherErr)
})
t.Run("the pinhole survives gateway discovery using its whole budget", func(t *testing.T) {
// On one shared context the probe would start already expired, which is
// how this failed against a real gateway.
reserve := 50 * time.Millisecond
origReserve := pinholeDiscoveryTimeout
pinholeDiscoveryTimeout = reserve
t.Cleanup(func() { pinholeDiscoveryTimeout = origReserve })
budget := 4 * reserve
ctx, cancel := context.WithTimeout(context.Background(), budget)
defer cancel()
stubDiscovery(t, nil, nat.ErrNoNATFound, budget, ipv6Pinhole, nil)
got, err := defaultDiscoverGateway(ctx)
require.NoError(t, err)
assert.Same(t, ipv6Pinhole, got)
})
}
func TestReserveForPinhole(t *testing.T) {
origReserve := pinholeDiscoveryTimeout
pinholeDiscoveryTimeout = time.Second
t.Cleanup(func() { pinholeDiscoveryTimeout = origReserve })
t.Run("a budget is divided", func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
gatewayCtx, cancelGateway := reserveForPinhole(ctx)
defer cancelGateway()
deadline, ok := gatewayCtx.Deadline()
require.True(t, ok)
assert.InDelta(t, 9*time.Second, time.Until(deadline), float64(500*time.Millisecond))
})
t.Run("a budget too small to divide is left whole", func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
gatewayCtx, cancelGateway := reserveForPinhole(ctx)
defer cancelGateway()
deadline, ok := gatewayCtx.Deadline()
require.True(t, ok)
assert.InDelta(t, 500*time.Millisecond, time.Until(deadline), float64(100*time.Millisecond))
})
t.Run("no deadline stays unbounded", func(t *testing.T) {
gatewayCtx, cancelGateway := reserveForPinhole(context.Background())
defer cancelGateway()
_, ok := gatewayCtx.Deadline()
assert.False(t, ok)
})
}

130
client/internal/profilemanager/prefs.go Normal file
View File

@@ -0,0 +1,130 @@
package profilemanager
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sync"
"github.com/netbirdio/netbird/util"
)
const prefsFileSuffix = ".prefs.json"
var prefsMu sync.Mutex
// Prefs is a namespaced per-profile preference store backed by a single JSON
// file next to the profile config; it is deleted together with the profile.
type Prefs struct {
path string
}
// ProfilePrefs returns the preference store of the profile identified by id.
func (s *ServiceManager) ProfilePrefs(id ID, username string) (*Prefs, error) {
if !IsValidProfileFilenameStem(id) {
return nil, fmt.Errorf("invalid profile ID: %q", id)
}
if id == defaultProfileName {
return &Prefs{path: filepath.Join(filepath.Dir(DefaultConfigPath), id.String()+prefsFileSuffix)}, nil
}
configDir, err := s.getConfigDir(username)
if err != nil {
return nil, fmt.Errorf("get config directory for user %s: %w", username, err)
}
return &Prefs{path: filepath.Join(configDir, id.String()+prefsFileSuffix)}, nil
}
// Get unmarshals the namespace section into v and reports whether it exists.
func (p *Prefs) Get(namespace string, v any) (bool, error) {
if namespace == "" {
return false, fmt.Errorf("empty prefs namespace")
}
prefsMu.Lock()
defer prefsMu.Unlock()
sections, err := readPrefsFile(p.path)
if err != nil {
return false, err
}
raw, ok := sections[namespace]
if !ok {
return false, nil
}
if err := json.Unmarshal(raw, v); err != nil {
return false, fmt.Errorf("decode prefs namespace %q: %w", namespace, err)
}
return true, nil
}
// Put stores v as the namespace section, replacing any previous value.
func (p *Prefs) Put(namespace string, v any) error {
if namespace == "" {
return fmt.Errorf("empty prefs namespace")
}
raw, err := json.Marshal(v)
if err != nil {
return fmt.Errorf("encode prefs namespace %q: %w", namespace, err)
}
prefsMu.Lock()
defer prefsMu.Unlock()
sections, err := readPrefsFile(p.path)
if err != nil {
return err
}
sections[namespace] = raw
return writePrefsFile(p.path, sections)
}
// Remove deletes the namespace section; a missing one is not an error.
func (p *Prefs) Remove(namespace string) error {
if namespace == "" {
return fmt.Errorf("empty prefs namespace")
}
prefsMu.Lock()
defer prefsMu.Unlock()
sections, err := readPrefsFile(p.path)
if err != nil {
return err
}
if _, ok := sections[namespace]; !ok {
return nil
}
delete(sections, namespace)
return writePrefsFile(p.path, sections)
}
func removePrefsFile(path string) error {
prefsMu.Lock()
defer prefsMu.Unlock()
return os.Remove(path)
}
func readPrefsFile(path string) (map[string]json.RawMessage, error) {
data, err := os.ReadFile(path)
if os.IsNotExist(err) {
return map[string]json.RawMessage{}, nil
}
if err != nil {
return nil, fmt.Errorf("read prefs: %w", err)
}
sections := map[string]json.RawMessage{}
if err := json.Unmarshal(data, &sections); err != nil {
return nil, fmt.Errorf("decode prefs: %w", err)
}
return sections, nil
}
func writePrefsFile(path string, sections map[string]json.RawMessage) error {
if err := util.WriteJsonWithRestrictedPermission(context.Background(), path, sections); err != nil {
return fmt.Errorf("write prefs: %w", err)
}
return nil
}

138
client/internal/profilemanager/prefs_test.go Normal file
View File

@@ -0,0 +1,138 @@
package profilemanager
import (
"errors"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type testPrefsSection struct {
Mode uint8 `json:"mode"`
Dest string `json:"dest"`
}
func TestProfilePrefs_RoundTrip(t *testing.T) {
withTestSM(t, func(sm *ServiceManager, username string) {
created, err := sm.AddProfile("work", username)
require.NoError(t, err)
prefs, err := sm.ProfilePrefs(created.ID, username)
require.NoError(t, err)
require.NoError(t, prefs.Put("filedrop", testPrefsSection{Mode: 2, Dest: "/tmp/x"}))
require.NoError(t, prefs.Put("other", map[string]int{"n": 1}))
var got testPrefsSection
found, err := prefs.Get("filedrop", &got)
require.NoError(t, err)
assert.True(t, found)
assert.Equal(t, testPrefsSection{Mode: 2, Dest: "/tmp/x"}, got)
var other map[string]int
found, err = prefs.Get("other", &other)
require.NoError(t, err)
assert.True(t, found)
assert.Equal(t, map[string]int{"n": 1}, other)
})
}
func TestProfilePrefs_GetMissingNamespace(t *testing.T) {
withTestSM(t, func(sm *ServiceManager, username string) {
created, err := sm.AddProfile("work", username)
require.NoError(t, err)
prefs, err := sm.ProfilePrefs(created.ID, username)
require.NoError(t, err)
var got testPrefsSection
found, err := prefs.Get("filedrop", &got)
require.NoError(t, err)
assert.False(t, found)
})
}
func TestProfilePrefs_RemoveNamespace(t *testing.T) {
withTestSM(t, func(sm *ServiceManager, username string) {
created, err := sm.AddProfile("work", username)
require.NoError(t, err)
prefs, err := sm.ProfilePrefs(created.ID, username)
require.NoError(t, err)
require.NoError(t, prefs.Put("filedrop", testPrefsSection{Mode: 1}))
require.NoError(t, prefs.Put("other", map[string]int{"n": 1}))
require.NoError(t, prefs.Remove("filedrop"))
require.NoError(t, prefs.Remove("missing"))
var got testPrefsSection
found, err := prefs.Get("filedrop", &got)
require.NoError(t, err)
assert.False(t, found)
var other map[string]int
found, err = prefs.Get("other", &other)
require.NoError(t, err)
assert.True(t, found)
assert.Equal(t, map[string]int{"n": 1}, other)
})
}
func TestProfilePrefs_RejectsInvalidID(t *testing.T) {
withTestSM(t, func(sm *ServiceManager, username string) {
_, err := sm.ProfilePrefs("../escape", username)
assert.Error(t, err)
})
}
func TestProfilePrefs_RejectsEmptyNamespace(t *testing.T) {
withTestSM(t, func(sm *ServiceManager, username string) {
created, err := sm.AddProfile("work", username)
require.NoError(t, err)
prefs, err := sm.ProfilePrefs(created.ID, username)
require.NoError(t, err)
_, err = prefs.Get("", &testPrefsSection{})
assert.Error(t, err)
assert.Error(t, prefs.Put("", testPrefsSection{}))
assert.Error(t, prefs.Remove(""))
})
}
func TestProfilePrefs_DefaultProfile(t *testing.T) {
withTestSM(t, func(sm *ServiceManager, username string) {
prefs, err := sm.ProfilePrefs(defaultProfileName, username)
require.NoError(t, err)
require.NoError(t, prefs.Put("filedrop", testPrefsSection{Mode: 1}))
expected := filepath.Join(filepath.Dir(DefaultConfigPath), "default"+prefsFileSuffix)
_, err = os.Stat(expected)
require.NoError(t, err)
})
}
func TestRemoveProfile_DeletesPrefsFile(t *testing.T) {
withTestSM(t, func(sm *ServiceManager, username string) {
created, err := sm.AddProfile("work", username)
require.NoError(t, err)
prefs, err := sm.ProfilePrefs(created.ID, username)
require.NoError(t, err)
require.NoError(t, prefs.Put("filedrop", testPrefsSection{Mode: 2}))
configDir, err := sm.getConfigDir(username)
require.NoError(t, err)
prefsPath := filepath.Join(configDir, created.ID.String()+prefsFileSuffix)
_, err = os.Stat(prefsPath)
require.NoError(t, err)
require.NoError(t, sm.RemoveProfile(created.ID, username))
_, err = os.Stat(prefsPath)
assert.True(t, errors.Is(err, os.ErrNotExist), "prefs file should be removed")
})
}

5
client/internal/profilemanager/service.go
View File

@@ -420,6 +420,11 @@ func (s *ServiceManager) RemoveProfile(id ID, username string) error {
log.Warnf("failed to remove profile state file %s: %v", stateFile, err)
}
prefsFile := filepath.Join(filepath.Dir(target.Path), id.String()+prefsFileSuffix)
if err := removePrefsFile(prefsFile); err != nil && !os.IsNotExist(err) {
log.Warnf("failed to remove profile prefs file %s: %v", prefsFile, err)
}
return nil
}

7
client/internal/profilemanager/state.go
View File

@@ -87,9 +87,10 @@ func (pm *ProfileManager) SetActiveProfileState(state *ProfileState) error {
// RemoveProfileState deletes the per-profile state file (which holds the
// account email used for the SSO login hint and the UI display). Called after
// a successful logout so a logged-out profile no longer shows a stale account
// email. The state file only stores the email, so deleting it is equivalent to
// clearing it; the next SSO login recreates it. A missing file is not an error.
// profile removal; logout keeps the file so the next login can pass the email
// as the login_hint. The state file only stores the email, so deleting it is
// equivalent to clearing it; the next SSO login recreates it. A missing file
// is not an error.
func (pm *ProfileManager) RemoveProfileState(profileName string) error {
configDir, err := getConfigDir()
if err != nil {

189
client/internal/routemanager/ipfwdstate/ipfwdstate.go
View File

@@ -2,54 +2,183 @@ package ipfwdstate
import (
"fmt"
"sync"
log "github.com/sirupsen/logrus"
"github.com/netbirdio/netbird/client/internal/routemanager/systemops"
)
// IPForwardingState is a struct that keeps track of the IP forwarding state.
// todo: read initial state of the IP forwarding from the system and reset the state based on it.
// todo: separate v4/v6 forwarding state, since the sysctls are independent
// (net.ipv4.ip_forward vs net.ipv6.conf.all.forwarding). Currently the nftables
// manager shares one instance between both routers, which works only because
// EnableIPForwarding enables both sysctls in a single call.
// IPForwardingState tracks v4 and v6 IP-forwarding sysctl enables with
// independent refcounts so a v4-only routing setup doesn't flip v6 sysctls.
type IPForwardingState struct {
enabledCounter int
mu sync.Mutex
v4Count int
v6Count int
// routingV4/routingV6 track whether the routing path currently holds a
// reference, so repeated EnableRouting calls (one per network-map update)
// hold at most one reference per family and an unpaired DisableRouting
// can't release references held by DNAT rules.
routingV4 bool
routingV6 bool
wgIfaceName string
v6Saved map[string]int
}
func NewIPForwardingState() *IPForwardingState {
return &IPForwardingState{}
// NewIPForwardingState returns a state tracker for the IP-forwarding sysctls.
// wgIfaceName is excluded from the per-interface accept_ra handling.
func NewIPForwardingState(wgIfaceName string) *IPForwardingState {
return &IPForwardingState{wgIfaceName: wgIfaceName}
}
func (f *IPForwardingState) RequestForwarding() error {
if f.enabledCounter != 0 {
f.enabledCounter++
// Counts returns the current v4 and v6 refcounts. Intended for diagnostics
// and tests.
func (f *IPForwardingState) Counts() (v4, v6 int) {
f.mu.Lock()
defer f.mu.Unlock()
return f.v4Count, f.v6Count
}
// RequestRouting takes the forwarding references for the routing path. It is
// idempotent: while routing already holds a reference, further calls don't
// increment the refcounts, and a v4-only request releases a previously held v6
// reference. A v6 sysctl failure is logged and not returned so it can't take
// down v4 routing (the sysctl may be unwritable, e.g. read-only /proc/sys or
// IPv6 disabled on the kernel command line); v6 is retried on the next call.
func (f *IPForwardingState) RequestRouting(v6 bool) error {
f.mu.Lock()
defer f.mu.Unlock()
if !f.routingV4 {
if err := f.requestV4(); err != nil {
return err
}
f.routingV4 = true
}
if !v6 {
if !f.routingV6 {
return nil
}
f.routingV6 = false
return f.releaseV6()
}
if f.routingV6 {
return nil
}
if err := systemops.EnableIPForwarding(); err != nil {
return fmt.Errorf("failed to enable IP forwarding with sysctl: %w", err)
if err := f.requestV6(); err != nil {
log.Warnf("enable IPv6 forwarding for routing: %v", err)
return nil
}
f.enabledCounter = 1
log.Info("IP forwarding enabled")
f.routingV6 = true
return nil
}
func (f *IPForwardingState) ReleaseForwarding() error {
if f.enabledCounter == 0 {
return nil
// ReleaseRouting releases the references RequestRouting holds. Calls without a
// held reference are no-ops.
func (f *IPForwardingState) ReleaseRouting() error {
f.mu.Lock()
defer f.mu.Unlock()
if f.routingV4 {
f.routingV4 = false
f.releaseV4()
}
if f.enabledCounter > 1 {
f.enabledCounter--
return nil
if f.routingV6 {
f.routingV6 = false
return f.releaseV6()
}
// if failed to disable IP forwarding we anyway decrement the counter
f.enabledCounter = 0
// todo call systemops.DisableIPForwarding()
return nil
}
// RequestForwarding enables the family's forwarding sysctl on first request.
func (f *IPForwardingState) RequestForwarding(v6 bool) error {
f.mu.Lock()
defer f.mu.Unlock()
if v6 {
return f.requestV6()
}
return f.requestV4()
}
// ReleaseForwarding decrements the family counter. The last v6 release restores
// what enable captured. v4 stays on: net.ipv4.ip_forward is co-owned by other
// tooling (docker, k8s, libvirt).
func (f *IPForwardingState) ReleaseForwarding(v6 bool) error {
f.mu.Lock()
defer f.mu.Unlock()
if v6 {
return f.releaseV6()
}
f.releaseV4()
return nil
}
func (f *IPForwardingState) requestV4() error {
if f.v4Count == 0 {
if err := systemops.EnableV4IPForwarding(); err != nil {
return fmt.Errorf("enable IPv4 forwarding: %w", err)
}
log.Info("IPv4 forwarding enabled")
}
f.v4Count++
return nil
}
func (f *IPForwardingState) releaseV4() {
if f.v4Count > 0 {
f.v4Count--
}
}
func (f *IPForwardingState) requestV6() error {
if f.v6Count == 0 {
saved, err := systemops.EnableV6IPForwarding(f.wgIfaceName)
if err != nil {
if rerr := systemops.DisableV6IPForwarding(saved); rerr != nil {
log.Warnf("rollback partial v6 sysctls: %v", rerr)
}
return fmt.Errorf("enable IPv6 forwarding: %w", err)
}
// A failed restore on a previous release keeps its saved values; those
// are the true originals, so keep them over what this enable captured.
if f.v6Saved == nil {
f.v6Saved = saved
} else {
for k, v := range saved {
if _, ok := f.v6Saved[k]; !ok {
f.v6Saved[k] = v
}
}
}
log.Info("IPv6 forwarding enabled")
}
f.v6Count++
return nil
}
func (f *IPForwardingState) releaseV6() error {
if f.v6Count == 0 {
return nil
}
f.v6Count--
if f.v6Count > 0 {
return nil
}
// Keep the saved values on failure so a later release or enable/release
// cycle can still restore them; re-restoring an already-restored key is a
// no-op since the sysctl already holds the desired value.
if err := systemops.DisableV6IPForwarding(f.v6Saved); err != nil {
return fmt.Errorf("disable IPv6 forwarding: %w", err)
}
f.v6Saved = nil
log.Info("IPv6 forwarding disabled")
return nil
}

39
client/internal/routemanager/ipfwdstate/ipfwdstate_privileged_linux_test.go Normal file
View File

@@ -0,0 +1,39 @@
//go:build privileged
package ipfwdstate
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestRequestRoutingV6ToV4Transition verifies that a v4-only routing request
// releases a previously held routing-owned v6 reference without touching
// references held by DNAT rules.
func TestRequestRoutingV6ToV4Transition(t *testing.T) {
f := NewIPForwardingState("wt-fwd-test")
require.NoError(t, f.RequestRouting(true), "request routing with v6")
v4, v6 := f.Counts()
assert.Equal(t, 1, v4, "v4 reference held")
assert.Equal(t, 1, v6, "v6 reference held")
require.NoError(t, f.RequestRouting(false), "request routing v4-only")
v4, v6 = f.Counts()
assert.Equal(t, 1, v4, "v4 reference kept")
assert.Equal(t, 0, v6, "routing-owned v6 reference released")
// A DNAT-held reference survives a v4-only routing request.
require.NoError(t, f.RequestForwarding(true), "dnat v6 reference")
require.NoError(t, f.RequestRouting(false), "repeat v4-only request")
_, v6 = f.Counts()
assert.Equal(t, 1, v6, "dnat-held v6 reference survives")
require.NoError(t, f.ReleaseForwarding(true), "release dnat v6 reference")
require.NoError(t, f.ReleaseRouting(), "release routing")
v4, v6 = f.Counts()
assert.Equal(t, 0, v4, "all v4 references released")
assert.Equal(t, 0, v6, "all v6 references released")
}

13
client/internal/routemanager/sysctl/sysctl_linux.go
View File

@@ -58,11 +58,7 @@ func Setup(wgIface iface) (map[string]int, error) {
continue
}
// Escape '%' and '.' so they survive the dot-to-slash conversion in Set()
safeName := strings.ReplaceAll(intf.Name, "%", percentEscape)
safeName = strings.ReplaceAll(safeName, ".", dotEscape)
i := fmt.Sprintf(rpFilterInterfacePath, safeName)
i := fmt.Sprintf(rpFilterInterfacePath, EscapeInterfaceName(intf.Name))
oldVal, err := Set(i, 2, true)
if err != nil {
result = multierror.Append(result, err)
@@ -74,6 +70,13 @@ func Setup(wgIface iface) (map[string]int, error) {
return keys, nberrors.FormatErrorOrNil(result)
}
// EscapeInterfaceName escapes '%' and '.' in an interface name (e.g. VLANs
// like eth0.100) so the name survives the dot-to-slash conversion in Set.
func EscapeInterfaceName(name string) string {
safe := strings.ReplaceAll(name, "%", percentEscape)
return strings.ReplaceAll(safe, ".", dotEscape)
}
// Set sets a sysctl configuration, if onlyIfOne is true it will only set the new value if it's set to 1
func Set(key string, desiredValue int, onlyIfOne bool) (int, error) {
path := strings.ReplaceAll(key, ".", "/")

82
client/internal/routemanager/systemops/routeselection_windows_test.go Normal file
View File

@@ -0,0 +1,82 @@
//go:build windows
package systemops
import (
"math"
"testing"
"github.com/stretchr/testify/assert"
)
func TestSortRouteCandidates(t *testing.T) {
tests := []struct {
name string
candidates []candidateRoute
wantOrder []uint32
}{
{
name: "longest prefix wins over metrics",
candidates: []candidateRoute{
{interfaceIndex: 1, prefixLength: 0, routeMetric: 0, interfaceMetric: 5},
{interfaceIndex: 2, prefixLength: 24, routeMetric: 100, interfaceMetric: 50},
},
wantOrder: []uint32{2, 1},
},
{
// Windows ranks equal-length prefixes by route metric + interface metric,
// so a higher route metric on a low metric interface can still win.
name: "combined metric beats route metric alone",
candidates: []candidateRoute{
{interfaceIndex: 8, prefixLength: 0, routeMetric: 0, interfaceMetric: 100},
{interfaceIndex: 5, prefixLength: 0, routeMetric: 10, interfaceMetric: 5},
},
wantOrder: []uint32{5, 8},
},
{
name: "lower combined metric wins",
candidates: []candidateRoute{
{interfaceIndex: 5, prefixLength: 0, routeMetric: 300, interfaceMetric: 5},
{interfaceIndex: 8, prefixLength: 0, routeMetric: 0, interfaceMetric: 100},
},
wantOrder: []uint32{8, 5},
},
{
name: "equal combined metric falls back to route metric",
candidates: []candidateRoute{
{interfaceIndex: 1, prefixLength: 0, routeMetric: 20, interfaceMetric: 10},
{interfaceIndex: 2, prefixLength: 0, routeMetric: 5, interfaceMetric: 25},
},
wantOrder: []uint32{2, 1},
},
{
// The metrics are uint32 on the Windows side, so the sum must not wrap.
name: "combined metric beyond the uint32 range",
candidates: []candidateRoute{
{interfaceIndex: 1, prefixLength: 0, routeMetric: math.MaxUint32, interfaceMetric: 5},
{interfaceIndex: 2, prefixLength: 0, routeMetric: math.MaxUint32 - 10, interfaceMetric: 5},
},
wantOrder: []uint32{2, 1},
},
{
name: "unknown interface metric ranks on route metric only",
candidates: []candidateRoute{
{interfaceIndex: 1, prefixLength: 0, routeMetric: 30, interfaceMetric: -1},
{interfaceIndex: 2, prefixLength: 0, routeMetric: 5, interfaceMetric: 10},
},
wantOrder: []uint32{2, 1},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sortRouteCandidates(tt.candidates)
got := make([]uint32, 0, len(tt.candidates))
for _, c := range tt.candidates {
got = append(got, c.interfaceIndex)
}
assert.Equal(t, tt.wantOrder, got)
})
}
}

13
client/internal/routemanager/systemops/systemops_android.go
View File

@@ -32,8 +32,17 @@ func (r *SysOps) removeFromRouteTable(netip.Prefix, Nexthop) error {
return nil
}
func EnableIPForwarding() error {
log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS)
func EnableV4IPForwarding() error {
log.Infof("Enable IPv4 forwarding is not implemented on %s", runtime.GOOS)
return nil
}
func EnableV6IPForwarding(string) (map[string]int, error) {
log.Infof("Enable IPv6 forwarding is not implemented on %s", runtime.GOOS)
return map[string]int{}, nil
}
func DisableV6IPForwarding(map[string]int) error {
return nil
}

13
client/internal/routemanager/systemops/systemops_ios.go
View File

@@ -58,8 +58,17 @@ func (r *SysOps) removeFromRouteTable(netip.Prefix, Nexthop) error {
return nil
}
func EnableIPForwarding() error {
log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS)
func EnableV4IPForwarding() error {
log.Infof("Enable IPv4 forwarding is not implemented on %s", runtime.GOOS)
return nil
}
func EnableV6IPForwarding(string) (map[string]int, error) {
log.Infof("Enable IPv6 forwarding is not implemented on %s", runtime.GOOS)
return map[string]int{}, nil
}
func DisableV6IPForwarding(map[string]int) error {
return nil
}

5
client/internal/routemanager/systemops/systemops_linux.go
View File

@@ -763,13 +763,10 @@ func flushRoutes(tableID, family int) error {
return nberrors.FormatErrorOrNil(result)
}
func EnableIPForwarding() error {
func EnableV4IPForwarding() error {
if _, err := sysctl.Set(ipv4ForwardingPath, 1, false); err != nil {
return err
}
if _, err := sysctl.Set(ipv6ForwardingPath, 1, false); err != nil {
log.Warnf("failed to enable IPv6 forwarding: %v", err)
}
return nil
}

13
client/internal/routemanager/systemops/systemops_nonlinux.go
View File

@@ -43,8 +43,17 @@ func (r *SysOps) RemoveVPNRoute(prefix netip.Prefix, intf *net.Interface) error
return r.genericRemoveVPNRoute(prefix, intf)
}
func EnableIPForwarding() error {
log.Infof("Enable IP forwarding is not implemented on %s", runtime.GOOS)
func EnableV4IPForwarding() error {
log.Infof("Enable IPv4 forwarding is not implemented on %s", runtime.GOOS)
return nil
}
func EnableV6IPForwarding(string) (map[string]int, error) {
log.Infof("Enable IPv6 forwarding is not implemented on %s", runtime.GOOS)
return map[string]int{}, nil
}
func DisableV6IPForwarding(map[string]int) error {
return nil
}

27
client/internal/routemanager/systemops/systemops_windows.go
View File

@@ -882,26 +882,40 @@ func getInterfaceMetric(interfaceIndex uint32, family int16) int {
return int(ipInterfaceRow.Metric)
}
// sortRouteCandidates sorts route candidates by priority: prefix length -> route metric -> interface metric
// sortRouteCandidates sorts route candidates by priority: prefix length -> combined metric -> route metric.
// Windows prefers the longest matching prefix and, among prefixes of the same length, the lowest metric, see
// https://learn.microsoft.com/en-us/windows-hardware/customize/desktop/unattend/microsoft-windows-tcpip-interfaces-interface-routes-route-metric
func sortRouteCandidates(candidates []candidateRoute) {
sort.Slice(candidates, func(i, j int) bool {
if candidates[i].prefixLength != candidates[j].prefixLength {
return candidates[i].prefixLength > candidates[j].prefixLength
}
if candidates[i].routeMetric != candidates[j].routeMetric {
return candidates[i].routeMetric < candidates[j].routeMetric
mi, mj := combinedMetric(candidates[i]), combinedMetric(candidates[j])
if mi != mj {
return mi < mj
}
return candidates[i].interfaceMetric < candidates[j].interfaceMetric
return candidates[i].routeMetric < candidates[j].routeMetric
})
}
// combinedMetric returns the effective metric Windows uses to rank routes with an equal prefix length:
// the sum of the route metric and the metric of the interface the route is on, see
// https://learn.microsoft.com/en-us/windows-server/networking/technologies/network-subsystem/net-sub-interface-metric
// An unknown interface metric contributes nothing.
func combinedMetric(candidate candidateRoute) uint64 {
if candidate.interfaceMetric < 0 {
return uint64(candidate.routeMetric)
}
return uint64(candidate.routeMetric) + uint64(candidate.interfaceMetric)
}
// GetBestInterface finds the best interface for reaching a destination,
// excluding the VPN interface to avoid routing loops.
//
// Route selection priority:
// 1. Longest prefix match (most specific route)
// 2. Lowest route metric
// 3. Lowest interface metric
// 2. Lowest combined metric (route metric + interface metric)
// 3. Lowest route metric.
func GetBestInterface(dest netip.Addr, vpnIntf string) (*net.Interface, error) {
var skipInterfaceIndex int
if vpnIntf != "" {
@@ -925,7 +939,6 @@ func GetBestInterface(dest netip.Addr, vpnIntf string) (*net.Interface, error) {
return nil, fmt.Errorf("no route to %s", dest)
}
// Sort routes: prefix length -> route metric -> interface metric
sortRouteCandidates(candidates)
for _, candidate := range candidates {

92
client/internal/routemanager/systemops/v6forwarding_linux.go Normal file
View File

@@ -0,0 +1,92 @@
//go:build !android
package systemops
import (
"fmt"
"net"
"os"
"github.com/hashicorp/go-multierror"
log "github.com/sirupsen/logrus"
nberrors "github.com/netbirdio/netbird/client/errors"
"github.com/netbirdio/netbird/client/internal/routemanager/sysctl"
)
const (
// 1 (default) accepts RAs only while forwarding is off; 2 keeps RA
// acceptance on regardless, so RA-installed host defaults survive our
// v6 forwarding flip.
acceptRAInterfacePath = "net.ipv6.conf.%s.accept_ra"
acceptRADefaultPath = "net.ipv6.conf.default.accept_ra"
acceptRAProcPathFormat = "/proc/sys/net/ipv6/conf/%s/accept_ra"
)
// EnableV6IPForwarding bumps accept_ra=2 on host v6 interfaces before flipping
// forwarding=1, so RA-installed host defaults survive. Returns the prior values
// of sysctls we actually changed; entries already at the target are omitted.
func EnableV6IPForwarding(wgIfaceName string) (map[string]int, error) {
saved := map[string]int{}
bumpAcceptRA(saved, wgIfaceName)
oldVal, err := sysctl.Set(ipv6ForwardingPath, 1, false)
if err != nil {
return saved, err
}
if oldVal != 1 {
saved[ipv6ForwardingPath] = oldVal
}
return saved, nil
}
// DisableV6IPForwarding restores what EnableV6IPForwarding captured.
func DisableV6IPForwarding(saved map[string]int) error {
var result *multierror.Error
for key, value := range saved {
if _, err := sysctl.Set(key, value, false); err != nil {
result = multierror.Append(result, fmt.Errorf("restore %s: %w", key, err))
}
}
return nberrors.FormatErrorOrNil(result)
}
func bumpAcceptRA(saved map[string]int, wgIfaceName string) {
// Also bump conf.default so interfaces created while forwarding is on
// (hotplug, new Wi-Fi/dock) inherit accept_ra=2 and keep accepting RAs.
bumpAcceptRAKey(saved, acceptRADefaultPath)
interfaces, err := net.Interfaces()
if err != nil {
log.Warnf("list interfaces for accept_ra: %v", err)
return
}
for _, intf := range interfaces {
if intf.Name == "lo" || intf.Name == wgIfaceName {
continue
}
bumpAcceptRAForInterface(saved, intf.Name)
}
}
func bumpAcceptRAForInterface(saved map[string]int, name string) {
// Build procfs path from name, not the dotted key: VLAN names like eth0.100.
if _, err := os.Stat(fmt.Sprintf(acceptRAProcPathFormat, name)); err != nil {
return
}
bumpAcceptRAKey(saved, fmt.Sprintf(acceptRAInterfacePath, sysctl.EscapeInterfaceName(name)))
}
func bumpAcceptRAKey(saved map[string]int, key string) {
// onlyIfOne=true: leave admin overrides (0, 2) alone.
oldVal, err := sysctl.Set(key, 2, true)
if err != nil {
log.Warnf("bump %s: %v", key, err)
return
}
// With onlyIfOne, a write only happened when the old value was 1; values
// left untouched (0, 2) must not be recorded for restore.
if oldVal == 1 {
saved[key] = oldVal
}
}

34
client/internal/routemanager/systemops/v6route_linux_test.go
View File

@@ -5,6 +5,7 @@ package systemops
import (
"errors"
"net"
"net/netip"
"syscall"
"testing"
@@ -29,6 +30,7 @@ func ensureIPv6DefaultRoute(t *testing.T) {
}
if err := netlink.RouteAdd(route); err != nil {
if errors.Is(err, syscall.EEXIST) {
requireUsableIPv6Nexthop(t)
return
}
t.Skipf("install IPv6 fallback default route: %v", err)
@@ -38,4 +40,36 @@ func ensureIPv6DefaultRoute(t *testing.T) {
t.Logf("delete IPv6 fallback default route: %v", err)
}
})
requireUsableIPv6Nexthop(t)
}
// requireUsableIPv6Nexthop skips the test unless the resolved IPv6 default
// nexthop can actually carry a route. Installing the default route succeeding
// does not imply the kernel accepts it as a nexthop for a concrete prefix.
func requireUsableIPv6Nexthop(t *testing.T) {
t.Helper()
nexthop, err := GetNextHop(netip.IPv6Unspecified())
if err != nil {
t.Skipf("resolve IPv6 default nexthop: %v", err)
}
probe := &netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
Table: syscall.RT_TABLE_MAIN,
Family: netlink.FAMILY_V6,
Dst: &net.IPNet{IP: net.ParseIP("100::64"), Mask: net.CIDRMask(128, 128)},
}
require.NoError(t, addNextHop(nexthop, probe), "build IPv6 probe route")
switch err := netlink.RouteAdd(probe); {
case err == nil:
if err := netlink.RouteDel(probe); err != nil && !errors.Is(err, syscall.ESRCH) {
t.Logf("delete IPv6 probe route: %v", err)
}
case errors.Is(err, syscall.EEXIST):
default:
t.Skipf("IPv6 nexthop %s unusable for route installation: %v", nexthop, err)
}
}

4
client/internal/sleep/service.go
View File

@@ -18,8 +18,8 @@ type Service struct {
}
func New() (*Service, error) {
d, err := NewDetector()
if err != nil {
d, err := NewDetector() //nolint:staticcheck
if err != nil { //nolint:staticcheck // always errors on platforms without a sleep detector
return nil, err
}

29
client/internal/updater/installer/doc.go
View File

@@ -37,23 +37,32 @@
// Updater Process (Setup):
//
// 1. Receives parameters from service via command-line arguments
// 2. Runs installer with appropriate silent/quiet flags:
// 2. Terminates the UI so the installer does not have to replace a locked image
// file, which would otherwise leave the install needing a reboot
// 3. Runs installer with appropriate silent/quiet flags:
// - Windows EXE: installer.exe /S
// - Windows MSI: msiexec.exe /i installer.msi /quiet /qn /l*v msi.log
// - Windows MSI: msiexec.exe /i installer.msi /qn /norestart REBOOT=ReallySuppress /l*v msi.log
// - macOS PKG: installer -pkg installer.pkg -target /
// - macOS Homebrew: brew upgrade netbirdio/tap/netbird
// 3. Installer terminates daemon and UI processes
// 4. Installer replaces binaries with new version
// 5. Updater waits for installer to complete
// 6. Updater restarts daemon:
// 4. Installer terminates the daemon
// 5. Installer replaces binaries with new version
// 6. Updater waits for installer to complete. On Windows, MSI exit codes 3010
// (ERROR_SUCCESS_REBOOT_REQUIRED) and 1641 (ERROR_SUCCESS_REBOOT_INITIATED)
// are a pending-reboot outcome, not a failure: the install succeeded, but
// some files are only replaced on the next restart (the reboot itself is
// suppressed via /norestart and REBOOT=ReallySuppress), and the flow
// continues as on success
// 7. Updater restarts daemon:
// - Windows: netbird.exe service start
// - macOS/Linux: netbird service start
// 7. Updater restarts UI:
// - Windows: Launches netbird-ui.exe as active console user using CreateProcessAsUser
// 8. Updater restarts UI:
// - Windows: Launches netbird-ui.exe using CreateProcessAsUser in every
// session it was terminated in, falling back to the active console session
// - macOS: Uses launchctl asuser to launch NetBird.app for console user
// - Linux: Not implemented (UI typically auto-starts)
// 8. Updater writes result.json with success/error status
// 9. Updater process exits
// 9. Updater writes result.json with success/error status (a pending reboot is
// recorded as success)
// 10. Updater process exits
//
// # Result Communication
//

3
client/internal/updater/installer/installer_common.go
View File

@@ -42,6 +42,9 @@ func NewWithDir(tempDir string) *Installer {
// This will run by the original service process
func (u *Installer) RunInstallation(ctx context.Context, targetVersion string) (err error) {
resultHandler := NewResultHandler(u.tempDir)
if err := resultHandler.ClearStaleResult(); err != nil {
log.Warnf("clear stale installer result: %v", err)
}
defer func() {
if err != nil {

182
client/internal/updater/installer/installer_run_windows.go
View File

@@ -2,6 +2,7 @@ package installer
import (
"context"
"errors"
"fmt"
"os"
"os/exec"
@@ -22,6 +23,12 @@ const (
msiLogFile = "msi.log"
// ERROR_SUCCESS_REBOOT_REQUIRED and ERROR_SUCCESS_REBOOT_INITIATED
msiRebootRequired = 3010
msiRebootInitiated = 1641
processExitWait = 10 * time.Second
msiDownloadURL = "https://github.com/netbirdio/netbird/releases/download/v%version/netbird_installer_%version_windows_%arch.msi"
exeDownloadURL = "https://github.com/netbirdio/netbird/releases/download/v%version/netbird_installer_%version_windows_%arch.exe"
)
@@ -38,6 +45,8 @@ var (
func (u *Installer) Setup(ctx context.Context, dryRun bool, installerFile string, daemonFolder string) (resultErr error) {
resultHandler := NewResultHandler(u.tempDir)
var uiSessions []uint32
// Always ensure daemon and UI are restarted after setup
defer func() {
log.Infof("starting daemon back")
@@ -46,7 +55,7 @@ func (u *Installer) Setup(ctx context.Context, dryRun bool, installerFile string
}
log.Infof("starting UI back")
if err := u.startUIAsUser(daemonFolder); err != nil {
if err := u.startUI(daemonFolder, uiSessions); err != nil {
log.Errorf("failed to start UI: %v", err)
}
@@ -75,6 +84,14 @@ func (u *Installer) Setup(ctx context.Context, dryRun bool, installerFile string
return
}
// The UI holds an open handle on its own image. Left running, Restart Manager
// cannot shut it down (msiexec runs as LocalSystem here, the UI as the
// interactive user), so the MSI falls back to replacing the file on reboot and
// marks the install as restart-required. The deferred close-application action
// in the package runs too late to prevent that, it happens after
// InstallValidate has already registered the file as in use.
uiSessions = killUI()
var cmd *exec.Cmd
switch installerType {
case TypeExe:
@@ -84,7 +101,9 @@ func (u *Installer) Setup(ctx context.Context, dryRun bool, installerFile string
installerDir := filepath.Dir(installerFile)
logPath := filepath.Join(installerDir, msiLogFile)
log.Infof("run msi installer: %s", installerFile)
cmd = exec.CommandContext(ctx, "msiexec.exe", "/i", filepath.Base(installerFile), "/quiet", "/qn", "/l*v", logPath)
// REBOOT=ReallySuppress: a silent install has no way to ask, so without it
// msiexec reboots the machine on its own if it decides one is needed.
cmd = exec.CommandContext(ctx, "msiexec.exe", "/i", filepath.Base(installerFile), "/qn", "/norestart", "REBOOT=ReallySuppress", "/l*v", logPath)
}
cmd.Dir = filepath.Dir(installerFile)
@@ -95,9 +114,13 @@ func (u *Installer) Setup(ctx context.Context, dryRun bool, installerFile string
}
log.Infof("installer started with PID %d", cmd.Process.Pid)
if resultErr = cmd.Wait(); resultErr != nil {
log.Errorf("installer process finished with error: %v", resultErr)
return
if err := cmd.Wait(); err != nil {
if !isRebootPending(err) {
resultErr = err
log.Errorf("installer process finished with error: %v", err)
return
}
log.Warnf("installer completed but reported a pending reboot, some files will be replaced on the next restart")
}
return nil
@@ -117,16 +140,142 @@ func (u *Installer) startDaemon(daemonFolder string) error {
return nil
}
func (u *Installer) startUIAsUser(daemonFolder string) error {
func (u *Installer) startUI(daemonFolder string, sessionIDs []uint32) error {
uiPath := filepath.Join(daemonFolder, uiName)
log.Infof("starting netbird-ui: %s", uiPath)
// Get the active console session ID
sessionID := windows.WTSGetActiveConsoleSessionId()
if sessionID == 0xFFFFFFFF {
return fmt.Errorf("no active user session found")
if len(sessionIDs) == 0 {
sessionID := windows.WTSGetActiveConsoleSessionId()
if sessionID == 0xFFFFFFFF {
return fmt.Errorf("no active user session found")
}
sessionIDs = []uint32{sessionID}
}
var errs []error
for _, sessionID := range sessionIDs {
if err := startUIInSession(uiPath, sessionID); err != nil {
errs = append(errs, fmt.Errorf("session %d: %w", sessionID, err))
continue
}
log.Infof("netbird-ui started successfully in session %d", sessionID)
}
return errors.Join(errs...)
}
// isRebootPending reports whether the installer exit code means it succeeded but
// left work for the next restart. The reboot itself is suppressed, so this is not
// a failure.
func isRebootPending(err error) bool {
var exitErr *exec.ExitError
if !errors.As(err, &exitErr) {
return false
}
switch exitErr.ExitCode() {
case msiRebootRequired, msiRebootInitiated:
return true
default:
return false
}
}
// killUI terminates any running netbird-ui process and returns the IDs of the
// interactive sessions the terminated processes belonged to. Setup starts the
// UI again in those sessions once the installer is done.
func killUI() []uint32 {
pids, err := processIDsByName(uiName)
if err != nil {
log.Warnf("failed to look up %s processes: %v", uiName, err)
return nil
}
sessions := make(map[uint32]struct{})
for _, pid := range pids {
var sessionID uint32
if err := windows.ProcessIdToSessionId(pid, &sessionID); err != nil {
log.Warnf("failed to look up session of %s (PID %d): %v", uiName, pid, err)
}
if err := terminateProcess(pid); err != nil {
log.Warnf("failed to terminate %s (PID %d): %v", uiName, pid, err)
continue
}
log.Infof("terminated %s (PID %d) in session %d", uiName, pid, sessionID)
if sessionID != 0 {
sessions[sessionID] = struct{}{}
}
}
sessionIDs := make([]uint32, 0, len(sessions))
for sessionID := range sessions {
sessionIDs = append(sessionIDs, sessionID)
}
return sessionIDs
}
func processIDsByName(name string) ([]uint32, error) {
snapshot, err := windows.CreateToolhelp32Snapshot(windows.TH32CS_SNAPPROCESS, 0)
if err != nil {
return nil, fmt.Errorf("create process snapshot: %w", err)
}
defer func() {
if err := windows.CloseHandle(snapshot); err != nil {
log.Warnf("failed to close process snapshot: %v", err)
}
}()
var entry windows.ProcessEntry32
entry.Size = uint32(unsafe.Sizeof(entry))
var pids []uint32
for err = windows.Process32First(snapshot, &entry); err == nil; err = windows.Process32Next(snapshot, &entry) {
if strings.EqualFold(windows.UTF16ToString(entry.ExeFile[:]), name) {
pids = append(pids, entry.ProcessID)
}
}
if !errors.Is(err, windows.ERROR_NO_MORE_FILES) {
return nil, fmt.Errorf("enumerate processes: %w", err)
}
return pids, nil
}
func terminateProcess(pid uint32) error {
handle, err := windows.OpenProcess(windows.PROCESS_TERMINATE|windows.SYNCHRONIZE, false, pid)
if err != nil {
// The process may have exited between enumeration and now.
if errors.Is(err, windows.ERROR_INVALID_PARAMETER) {
return nil
}
return fmt.Errorf("open process: %w", err)
}
defer func() {
if err := windows.CloseHandle(handle); err != nil {
log.Warnf("failed to close process handle: %v", err)
}
}()
if err := windows.TerminateProcess(handle, 0); err != nil {
return fmt.Errorf("terminate process: %w", err)
}
// Wait for the handle to signal so the image file is released before the
// installer tries to overwrite it. A timeout is reported through the returned
// event, not through err, which stays nil unless the wait itself failed.
event, err := windows.WaitForSingleObject(handle, uint32(processExitWait.Milliseconds()))
if err != nil {
return fmt.Errorf("wait for process exit: %w", err)
}
if event != windows.WAIT_OBJECT_0 {
return fmt.Errorf("wait for process exit: unexpected wait result %#x", event)
}
return nil
}
func startUIInSession(uiPath string, sessionID uint32) error {
// Get the user token for that session
var userToken windows.Token
err := windows.WTSQueryUserToken(sessionID, &userToken)
@@ -158,6 +307,16 @@ func (u *Installer) startUIAsUser(daemonFolder string) error {
}
}()
var env *uint16
if err := windows.CreateEnvironmentBlock(&env, primaryToken, false); err != nil {
return fmt.Errorf("create environment block: %w", err)
}
defer func() {
if err := windows.DestroyEnvironmentBlock(env); err != nil {
log.Warnf("failed to destroy environment block: %v", err)
}
}()
// Prepare startup info
var si windows.StartupInfo
si.Cb = uint32(unsafe.Sizeof(si))
@@ -180,7 +339,7 @@ func (u *Installer) startUIAsUser(daemonFolder string) error {
nil,
false,
creationFlags,
nil,
env,
nil,
&si,
&pi,
@@ -197,7 +356,6 @@ func (u *Installer) startUIAsUser(daemonFolder string) error {
log.Warnf("failed to close thread handle: %v", err)
}
log.Infof("netbird-ui started successfully in session %d", sessionID)
return nil
}

108
client/internal/updater/installer/installer_run_windows_test.go Normal file
View File

@@ -0,0 +1,108 @@
package installer
import (
"errors"
"os/exec"
"slices"
"strconv"
"testing"
)
// exitErrorWithCode returns a real *exec.ExitError carrying the given exit code.
func exitErrorWithCode(t *testing.T, code int) error {
t.Helper()
err := exec.Command("cmd.exe", "/c", "exit "+strconv.Itoa(code)).Run()
if err == nil {
t.Fatalf("expected a non-zero exit for code %d", code)
}
return err
}
func TestIsRebootPending(t *testing.T) {
tests := []struct {
name string
code int
want bool
}{
{name: "reboot required", code: msiRebootRequired, want: true},
{name: "reboot initiated", code: msiRebootInitiated, want: true},
{name: "generic failure", code: 1603, want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := isRebootPending(exitErrorWithCode(t, tt.code)); got != tt.want {
t.Errorf("isRebootPending(exit %d) = %v, want %v", tt.code, got, tt.want)
}
})
}
}
// TestProcessIDsByNameAndTerminate spawns a long-running system process, finds it
// by name and terminates it, covering the path the updater uses to release the UI
// image file before the installer replaces it.
func TestProcessIDsByNameAndTerminate(t *testing.T) {
cmd := exec.Command("ping.exe", "-n", "60", "127.0.0.1")
if err := cmd.Start(); err != nil {
t.Fatalf("start ping: %v", err)
}
pid := uint32(cmd.Process.Pid)
killed := false
t.Cleanup(func() {
if !killed {
_ = cmd.Process.Kill()
}
_ = cmd.Wait()
})
// Name matching must be case-insensitive: the snapshot reports PING.EXE.
pids, err := processIDsByName("ping.exe")
if err != nil {
t.Fatalf("processIDsByName: %v", err)
}
if !slices.Contains(pids, pid) {
t.Fatalf("PID %d not among the ping.exe processes found: %v", pid, pids)
}
if err := terminateProcess(pid); err != nil {
t.Fatalf("terminateProcess: %v", err)
}
killed = true
// terminateProcess only returns once the handle has signalled, so the process
// is already gone and Wait must not block. It exits with the code passed to
// TerminateProcess, which is 0, so Wait reports no error.
if err := cmd.Wait(); err != nil {
t.Fatalf("wait for terminated ping: %v", err)
}
if !cmd.ProcessState.Exited() {
t.Error("process did not exit after terminateProcess")
}
remaining, err := processIDsByName("ping.exe")
if err != nil {
t.Fatalf("processIDsByName after terminate: %v", err)
}
if slices.Contains(remaining, pid) {
t.Errorf("PID %d still listed after terminateProcess", pid)
}
}
func TestProcessIDsByNameNoMatch(t *testing.T) {
pids, err := processIDsByName("netbird-nonexistent-process.exe")
if err != nil {
t.Fatalf("processIDsByName: %v", err)
}
if len(pids) != 0 {
t.Errorf("expected no matches, got %v", pids)
}
}
func TestIsRebootPendingNonExitError(t *testing.T) {
if isRebootPending(errors.New("start installer: file not found")) {
t.Error("a non-exit error must not be treated as a pending reboot")
}
}

6
client/internal/updater/installer/result.go
View File

@@ -54,6 +54,12 @@ func (rh *ResultHandler) GetErrorResultReason() string {
return ""
}
// ClearStaleResult removes a result file left over from a previous installation
// attempt so result watchers cannot read an outdated outcome for the current attempt.
func (rh *ResultHandler) ClearStaleResult() error {
return rh.cleanup()
}
func (rh *ResultHandler) WriteSuccess() error {
result := Result{
Success: true,

2
client/internal/updater/manager.go
View File

@@ -435,7 +435,7 @@ func (m *Manager) install(ctx context.Context, pendingVersion *v.Version) error
}
inst := installer.New()
if err := inst.RunInstallation(ctx, pendingVersion.String()); err != nil {
if err := inst.RunInstallation(ctx, pendingVersion.String()); err != nil { //nolint:staticcheck // always errors on platforms without an installer
log.Errorf("error triggering update: %v", err)
m.statusRecorder.PublishEvent(
cProto.SystemEvent_ERROR,