Compare commits

..

2 Commits

Author SHA1 Message Date
riccardom
aa4257f1ac [client] make AllowedIPs swap self-healing on WireGuard errors
Address CodeRabbit review on #6799.

Decrement now decides whether to reprogram based on whether the currently
installed peer still holds references (e.peers[e.active] > 0) instead of
whether the released peer was the active one. If a prior swap's remove/add
failed, e.active was left pointing at a peer with zero references and every
later Decrement returned early, permanently stranding the prefix and leaking
the entry. The active peer is now detached only when e.active != "", and a
failed hand-off is retried on the next Decrement/Increment.

Also clear currentPeerKey unconditionally in the static handler's
RemoveAllowedIPs (matching dynamic/dnsinterceptor) and assert Increment
errors in the tests. Added self-heal tests for the failed remove/add paths.
2026-07-16 16:07:52 +02:00
riccardom
f6a756c962 [client] fix stale routing peer on overlapping-prefix network removal
Make the AllowedIPs refcounter peer-aware: track per-peer refcounts per
  prefix and swap WireGuard to a surviving peer when the installed one is
  released, instead of leaving the prefix on the removed peer. Fixes routing
  peer not updating without netbird down/up when two networks share a prefix.
2026-07-16 14:07:40 +02:00
20 changed files with 538 additions and 406 deletions

View File

@@ -17,7 +17,8 @@ ENV \
NETBIRD_BIN="/usr/local/bin/netbird" \
NB_LOG_FILE="console,/var/log/netbird/client.log" \
NB_DAEMON_ADDR="unix:///var/run/netbird.sock" \
NB_ENABLE_CAPTURE="false"
NB_ENABLE_CAPTURE="false" \
NB_ENTRYPOINT_SERVICE_TIMEOUT="30"
ENTRYPOINT [ "/usr/local/bin/netbird-entrypoint.sh" ]
ARG TARGETPLATFORM

View File

@@ -23,7 +23,8 @@ ENV \
NB_DAEMON_ADDR="unix:///var/lib/netbird/netbird.sock" \
NB_LOG_FILE="console,/var/lib/netbird/client.log" \
NB_DISABLE_DNS="true" \
NB_ENABLE_CAPTURE="false"
NB_ENABLE_CAPTURE="false" \
NB_ENTRYPOINT_SERVICE_TIMEOUT="30"
ENTRYPOINT [ "/usr/local/bin/netbird-entrypoint.sh" ]
ARG TARGETPLATFORM

View File

@@ -1,176 +0,0 @@
package cmd
import (
"context"
"net"
"path/filepath"
"sync/atomic"
"testing"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/connectivity"
"github.com/netbirdio/netbird/client/internal"
"github.com/netbirdio/netbird/client/proto"
)
// startUnixGRPCServer starts a bare gRPC server listening on a unix socket at path
// and returns a stop function. No services are registered; the connectivity-state
// wait only cares about the transport becoming READY.
func startUnixGRPCServer(t *testing.T, path string) func() {
t.Helper()
lis, err := net.Listen("unix", path)
if err != nil {
t.Fatalf("listen unix %s: %v", path, err)
}
srv := grpc.NewServer()
go func() { _ = srv.Serve(lis) }()
return srv.Stop
}
func TestDialClientGRPCServer_ConnectsWhenServing(t *testing.T) {
sock := filepath.Join(t.TempDir(), "nb.sock")
stop := startUnixGRPCServer(t, sock)
defer stop()
conn, err := dialClientGRPCServer(context.Background(), "unix://"+sock, 5*time.Second)
if err != nil {
t.Fatalf("expected connection, got error: %v", err)
}
defer conn.Close()
if state := conn.GetState(); state != connectivity.Ready {
t.Fatalf("expected READY, got %s", state)
}
}
// TestDialClientGRPCServer_WaitsForLateServer is the core regression test: the
// daemon socket appears only after the dial has already started, mirroring
// "netbird service start" immediately followed by "netbird up".
func TestDialClientGRPCServer_WaitsForLateServer(t *testing.T) {
sock := filepath.Join(t.TempDir(), "nb.sock")
var stop func()
timer := time.AfterFunc(1*time.Second, func() {
stop = startUnixGRPCServer(t, sock)
})
defer timer.Stop()
defer func() {
if stop != nil {
stop()
}
}()
start := time.Now()
conn, err := dialClientGRPCServer(context.Background(), "unix://"+sock, 10*time.Second)
if err != nil {
t.Fatalf("expected connection after late server start, got error: %v", err)
}
defer conn.Close()
if elapsed := time.Since(start); elapsed < 500*time.Millisecond {
t.Fatalf("connected too fast (%s); server should not have been up yet", elapsed)
}
if state := conn.GetState(); state != connectivity.Ready {
t.Fatalf("expected READY, got %s", state)
}
}
// fakeStatusServer serves the Status RPC with a programmable response so we can
// exercise waitForDaemonStatus without spinning up a real engine.
type fakeStatusServer struct {
proto.UnimplementedDaemonServiceServer
resp func() *proto.StatusResponse
}
func (f *fakeStatusServer) Status(context.Context, *proto.StatusRequest) (*proto.StatusResponse, error) {
return f.resp(), nil
}
func startFakeStatusServer(t *testing.T, sock string, resp func() *proto.StatusResponse) func() {
t.Helper()
lis, err := net.Listen("unix", sock)
if err != nil {
t.Fatalf("listen unix %s: %v", sock, err)
}
srv := grpc.NewServer()
proto.RegisterDaemonServiceServer(srv, &fakeStatusServer{resp: resp})
go func() { _ = srv.Serve(lis) }()
return srv.Stop
}
func dialFake(t *testing.T, sock string) proto.DaemonServiceClient {
t.Helper()
conn, err := dialClientGRPCServer(context.Background(), "unix://"+sock, 5*time.Second)
if err != nil {
t.Fatalf("dial fake daemon: %v", err)
}
t.Cleanup(func() { conn.Close() })
return proto.NewDaemonServiceClient(conn)
}
// New daemon that flips DaemonReady=true after a couple of polls: waitForDaemonStatus
// must block until the flag is set, then return.
func TestWaitForDaemonStatus_WaitsForDaemonReady(t *testing.T) {
sock := filepath.Join(t.TempDir(), "nb.sock")
var polls int32
stop := startFakeStatusServer(t, sock, func() *proto.StatusResponse {
n := atomic.AddInt32(&polls, 1)
return &proto.StatusResponse{
Status: string(internal.StatusConnecting),
DaemonReady: n >= 3, // ready only from the 3rd poll on
}
})
defer stop()
client := dialFake(t, sock)
status, err := waitForDaemonStatus(context.Background(), client)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !status.GetDaemonReady() {
t.Fatalf("expected DaemonReady=true, got false")
}
if got := atomic.LoadInt32(&polls); got < 3 {
t.Fatalf("expected at least 3 polls before ready, got %d", got)
}
}
// Older daemon that never sets DaemonReady but reports a healthy (Connected)
// status: waitForDaemonStatus must return promptly via the readiness fallback,
// not block for the whole grace window.
func TestWaitForDaemonStatus_OlderDaemonHealthyStatus(t *testing.T) {
sock := filepath.Join(t.TempDir(), "nb.sock")
stop := startFakeStatusServer(t, sock, func() *proto.StatusResponse {
return &proto.StatusResponse{Status: string(internal.StatusConnected)} // DaemonReady unset
})
defer stop()
client := dialFake(t, sock)
start := time.Now()
status, err := waitForDaemonStatus(context.Background(), client)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if status.GetDaemonReady() {
t.Fatalf("expected DaemonReady=false from older daemon")
}
if elapsed := time.Since(start); elapsed > 2*time.Second {
t.Fatalf("returned too slowly (%s); healthy status should short-circuit the grace", elapsed)
}
}
func TestDialClientGRPCServer_TimesOutWhenAbsent(t *testing.T) {
sock := filepath.Join(t.TempDir(), "never.sock")
start := time.Now()
conn, err := dialClientGRPCServer(context.Background(), "unix://"+sock, 1*time.Second)
if err == nil {
conn.Close()
t.Fatal("expected timeout error, got nil")
}
if elapsed := time.Since(start); elapsed < 900*time.Millisecond {
t.Fatalf("returned too early (%s); should have waited ~timeout", elapsed)
}
}

View File

@@ -20,8 +20,6 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"google.golang.org/grpc"
gbackoff "google.golang.org/grpc/backoff"
"google.golang.org/grpc/connectivity"
"google.golang.org/grpc/credentials/insecure"
daddr "github.com/netbirdio/netbird/client/internal/daemonaddr"
@@ -266,70 +264,17 @@ func FlagNameToEnvVar(cmdFlag string, prefix string) string {
return prefix + upper
}
// defaultDaemonDialTimeout is how long DialClientGRPCServer waits for the daemon
// to become reachable. It is intentionally generous so that invoking the CLI
// right after "netbird service start" (e.g. from a container entrypoint) tolerates
// the window where the daemon has created its socket but is not yet serving.
const defaultDaemonDialTimeout = 30 * time.Second
// DialClientGRPCServer returns a client connection to the daemon server. It waits
// for the daemon to become reachable, retrying with backoff until the connection
// reports READY or defaultDaemonDialTimeout elapses. This handles the startup race
// where the daemon socket exists (or is about to) before the gRPC server is serving.
// DialClientGRPCServer returns client connection to the daemon server.
func DialClientGRPCServer(ctx context.Context, addr string) (*grpc.ClientConn, error) {
return dialClientGRPCServer(ctx, addr, defaultDaemonDialTimeout)
}
func dialClientGRPCServer(ctx context.Context, addr string, timeout time.Duration) (*grpc.ClientConn, error) {
conn, err := grpc.NewClient(
strings.TrimPrefix(addr, "tcp://"),
grpc.WithTransportCredentials(insecure.NewCredentials()),
// Cap reconnect backoff at 5s; gRPC's default 120s MaxDelay would leave the
// CLI waiting far too long to notice a freshly-started daemon. Mirrors the GUI.
grpc.WithConnectParams(grpc.ConnectParams{
Backoff: gbackoff.Config{
BaseDelay: 1 * time.Second,
Multiplier: 1.6,
Jitter: 0.2,
MaxDelay: 5 * time.Second,
},
}),
)
if err != nil {
return nil, fmt.Errorf("create daemon gRPC client: %w", err)
}
// grpc.NewClient is lazy: it does not connect until the first RPC or until we
// nudge it. Trigger connection attempts and wait until the channel reaches READY.
if err := waitForConnReady(ctx, conn, timeout); err != nil {
_ = conn.Close()
return nil, err
}
return conn, nil
}
// waitForConnReady drives the gRPC channel out of IDLE and blocks until it becomes
// READY, or until timeout/ctx expires. TRANSIENT_FAILURE (daemon not yet serving)
// is treated as retryable so the caller keeps waiting within the deadline.
func waitForConnReady(ctx context.Context, conn *grpc.ClientConn, timeout time.Duration) error {
ctx, cancel := context.WithTimeout(ctx, timeout)
ctx, cancel := context.WithTimeout(ctx, time.Second*10)
defer cancel()
for {
state := conn.GetState()
switch state {
case connectivity.Ready:
return nil
case connectivity.Idle:
// Kick the lazy channel into connecting.
conn.Connect()
}
if !conn.WaitForStateChange(ctx, state) {
// ctx expired while in `state`.
return fmt.Errorf("timed out after %s waiting for daemon to become ready (last state: %s)", timeout, state)
}
}
return grpc.DialContext(
ctx,
strings.TrimPrefix(addr, "tcp://"),
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithBlock(),
)
}
// WithBackOff execute function in backoff cycle.

View File

@@ -78,10 +78,6 @@ func (p *program) Start(svc service.Service) error {
log.Fatalf("failed to start daemon: %v", err)
}
proto.RegisterDaemonServiceServer(p.serv, serverInstance)
// The engine is started and the service is registered: from here on the
// daemon serves RPCs backed by a running engine. Report readiness so
// clients (e.g. netbird up) can wait deterministically.
serverInstance.SetReady()
p.serverInstanceMu.Lock()
p.serverInstance = serverInstance

View File

@@ -295,7 +295,9 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command, pm *profilemanager
client := proto.NewDaemonServiceClient(conn)
status, err := waitForDaemonStatus(ctx, client)
status, err := client.Status(ctx, &proto.StatusRequest{
WaitForReady: func() *bool { b := true; return &b }(),
})
if err != nil {
return fmt.Errorf("unable to get daemon status: %v", err)
}
@@ -334,79 +336,6 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command, pm *profilemanager
return nil
}
// daemonStatusPollTimeout bounds how long we poll the Status RPC waiting for the
// daemon to answer coherently. The transport is already READY at this point (see
// DialClientGRPCServer), so this only covers the brief window where the gRPC server
// is serving but the daemon engine is still starting up and the Status RPC races
// against server.Start().
const daemonStatusPollTimeout = 15 * time.Second
// daemonReadyGrace bounds how long we keep polling once the daemon answers but
// still reports DaemonReady=false. A freshly-started daemon flips it to true
// within this window; an older daemon that never sets the field simply falls
// through after the grace elapses, preserving backward compatibility.
const daemonReadyGrace = 10 * time.Second
// waitForDaemonStatus fetches the daemon status, waiting for the daemon to become
// ready. It handles two startup races:
//
// 1. The gRPC server is not yet serving: Status fails with Unavailable — retry.
// 2. The server serves but the engine is still starting: a DaemonReady-aware
// daemon reports DaemonReady=false until Start finishes; poll until it flips
// true (bounded by daemonReadyGrace). Older daemons never set DaemonReady, so
// we stop waiting on it after the grace and use the status as-is.
//
// It gives up after daemonStatusPollTimeout.
func waitForDaemonStatus(ctx context.Context, client proto.DaemonServiceClient) (*proto.StatusResponse, error) {
ctx, cancel := context.WithTimeout(ctx, daemonStatusPollTimeout)
defer cancel()
waitForReady := true
req := &proto.StatusRequest{WaitForReady: &waitForReady}
var lastErr error
var firstAnswer time.Time
for {
status, err := client.Status(ctx, req)
if err != nil {
lastErr = err
// Only retry while the daemon is not yet answering; surface real errors.
if s, ok := gstatus.FromError(err); !ok || s.Code() != codes.Unavailable {
return nil, err
}
} else {
// Daemon answered. Explicitly ready (DaemonReady-aware daemon), or
// already fully connected — either way, done. Connected is the only
// status unambiguous enough to short-circuit on: a DaemonReady-aware
// daemon sets the flag at startup, so trusting Connected here can only
// help an older daemon that never sets the flag, without overriding a
// new daemon that legitimately reports DaemonReady=false while starting.
if status.GetDaemonReady() || internal.StatusType(status.GetStatus()) == internal.StatusConnected {
return status, nil
}
// Answered but neither ready-flagged nor connected yet: give a
// DaemonReady-aware daemon a bounded window to finish starting, then
// fall through so an older daemon that never sets the flag isn't
// blocked here.
if firstAnswer.IsZero() {
firstAnswer = time.Now()
} else if time.Since(firstAnswer) >= daemonReadyGrace {
return status, nil
}
lastErr = nil
}
select {
case <-ctx.Done():
if lastErr != nil {
return nil, fmt.Errorf("daemon did not become ready: %w", lastErr)
}
return nil, fmt.Errorf("daemon did not become ready within %s", daemonStatusPollTimeout)
case <-time.After(500 * time.Millisecond):
}
}
}
func doDaemonUp(ctx context.Context, cmd *cobra.Command, client proto.DaemonServiceClient, pm *profilemanager.ProfileManager, activeProf *profilemanager.Profile, customDNSAddressConverted []byte, username string) error {
providedSetupKey, err := getSetupKey()

View File

@@ -292,16 +292,18 @@ func (s *serviceViaListener) generateFreePort() (uint16, error) {
return customPort, nil
}
probeListener, err := net.ListenUDP("udp4", &net.UDPAddr{})
udpAddr := net.UDPAddrFromAddrPort(netip.MustParseAddrPort("0.0.0.0:0"))
probeListener, err := net.ListenUDP("udp", udpAddr)
if err != nil {
log.Debugf("failed to bind random port for DNS: %s", err)
return 0, err
}
port := uint16(probeListener.LocalAddr().(*net.UDPAddr).Port)
if err = probeListener.Close(); err != nil {
addrPort := netip.MustParseAddrPort(probeListener.LocalAddr().String()) // might panic if address is incorrect
err = probeListener.Close()
if err != nil {
log.Debugf("failed to free up DNS port: %s", err)
return 0, err
}
return port, nil
return addrPort.Port(), nil
}

View File

@@ -95,7 +95,7 @@ func (d *DnsInterceptor) RemoveRoute() error {
// AllowedIPs should use real IPs
if d.currentPeerKey != "" {
if _, err := d.allowedIPsRefcounter.Decrement(prefix); err != nil {
if _, err := d.allowedIPsRefcounter.Decrement(prefix, d.currentPeerKey); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %v", prefix, err))
}
}
@@ -172,7 +172,7 @@ func (d *DnsInterceptor) removeAllowedIP(realPrefix netip.Prefix) error {
}
// AllowedIPs use real IPs
if _, err := d.allowedIPsRefcounter.Decrement(realPrefix); err != nil {
if _, err := d.allowedIPsRefcounter.Decrement(realPrefix, d.currentPeerKey); err != nil {
return fmt.Errorf("remove allowed IP %s: %v", realPrefix, err)
}
@@ -205,7 +205,7 @@ func (d *DnsInterceptor) RemoveAllowedIPs() error {
for _, prefixes := range d.interceptedDomains {
for _, prefix := range prefixes {
// AllowedIPs use real IPs
if _, err := d.allowedIPsRefcounter.Decrement(prefix); err != nil {
if _, err := d.allowedIPsRefcounter.Decrement(prefix, d.currentPeerKey); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %v", prefix, err))
}
}

View File

@@ -135,7 +135,7 @@ func (r *Route) RemoveAllowedIPs() error {
var merr *multierror.Error
for _, domainPrefixes := range r.dynamicDomains {
for _, prefix := range domainPrefixes {
if _, err := r.allowedIPsRefcounter.Decrement(prefix); err != nil {
if _, err := r.allowedIPsRefcounter.Decrement(prefix, r.currentPeerKey); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %w", prefix, err))
}
}
@@ -320,7 +320,7 @@ func (r *Route) removeRoutes(prefixes []netip.Prefix) ([]netip.Prefix, error) {
merr = multierror.Append(merr, fmt.Errorf("remove dynamic route for IP %s: %w", prefix, err))
}
if r.currentPeerKey != "" {
if _, err := r.allowedIPsRefcounter.Decrement(prefix); err != nil {
if _, err := r.allowedIPsRefcounter.Decrement(prefix, r.currentPeerKey); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %w", prefix, err))
}
}

View File

@@ -215,7 +215,7 @@ func (m *DefaultManager) setupRefCounters(useNoop bool) {
)
}
m.allowedIPsRefCounter = refcounter.New(
m.allowedIPsRefCounter = refcounter.NewAllowedIPs(
func(prefix netip.Prefix, peerKey string) (string, error) {
// save peerKey to use it in the remove function
return peerKey, m.wgInterface.AddAllowedIP(peerKey, prefix)

View File

@@ -0,0 +1,185 @@
package refcounter
import (
"errors"
"fmt"
"net/netip"
"sort"
"sync"
"github.com/hashicorp/go-multierror"
nberrors "github.com/netbirdio/netbird/client/errors"
)
// allowedIPsEntry holds the per-peer reference counts for a single prefix and which peer is
// currently installed in WireGuard. WireGuard allows a prefix on exactly one peer, so at most
// one peer is active at a time even when several peers reference the prefix.
type allowedIPsEntry struct {
// peers maps a peerKey to the number of references holding the prefix for that peer.
peers map[string]int
// active is the peerKey currently installed in WireGuard for this prefix ("" if none).
active string
// total is the sum of all per-peer reference counts (kept in sync with peers).
total int
}
// AllowedIPsRefCounter is a peer-aware reference counter for WireGuard AllowedIPs.
//
// The generic Counter keys only by prefix and remembers a single Out value set by the first
// caller, which it never changes. That is wrong for AllowedIPs: two independent watchers (or
// multiple resolved domains) can reference the same prefix through different peers, and when the
// peer currently installed in WireGuard releases its last reference the prefix must be handed over
// to a surviving peer instead of being left pointing at the released one.
//
// It calls add/remove (which program WireGuard) only on the transitions that matter:
// - add on the first reference for a prefix, or when swapping the active peer;
// - remove on the last reference for a prefix, or on the old peer during a swap.
type AllowedIPsRefCounter struct {
mu sync.Mutex
entries map[netip.Prefix]*allowedIPsEntry
add AddFunc[netip.Prefix, string, string]
remove RemoveFunc[netip.Prefix, string]
}
// NewAllowedIPs creates a new peer-aware AllowedIPs reference counter.
// add programs a prefix on a peer in WireGuard and returns the peerKey to store as the active peer.
// remove unprograms the prefix from the given peer.
func NewAllowedIPs(add AddFunc[netip.Prefix, string, string], remove RemoveFunc[netip.Prefix, string]) *AllowedIPsRefCounter {
return &AllowedIPsRefCounter{
entries: map[netip.Prefix]*allowedIPsEntry{},
add: add,
remove: remove,
}
}
// Increment adds a reference to prefix for peerKey. WireGuard is programmed only for the first
// reference to a prefix; while a different peer is already installed the prefix is left with it
// (first peer wins, HA at the WireGuard layer is not possible) and only the reference count is kept.
func (rm *AllowedIPsRefCounter) Increment(prefix netip.Prefix, peerKey string) (Ref[string], error) {
rm.mu.Lock()
defer rm.mu.Unlock()
e, ok := rm.entries[prefix]
if !ok {
e = &allowedIPsEntry{peers: map[string]int{}}
rm.entries[prefix] = e
}
logCallerF("Increasing allowed IP ref count for prefix %v peer %s [peer %d -> %d, total %d -> %d, active %q]",
prefix, peerKey, e.peers[peerKey], e.peers[peerKey]+1, e.total, e.total+1, e.active)
// Program WireGuard only when nothing is installed yet for this prefix.
if e.active == "" {
out, err := rm.add(prefix, peerKey)
if errors.Is(err, ErrIgnore) {
if e.total == 0 {
delete(rm.entries, prefix)
}
return Ref[string]{Count: e.total, Out: e.active}, nil
}
if err != nil {
if e.total == 0 {
delete(rm.entries, prefix)
}
return Ref[string]{}, fmt.Errorf("failed to add allowed IP %v for peer %s: %w", prefix, peerKey, err)
}
e.active = out
}
e.peers[peerKey]++
e.total++
return Ref[string]{Count: e.total, Out: e.active}, nil
}
// Decrement removes a reference to prefix for peerKey. When the peer currently installed in
// WireGuard releases its last reference, the prefix is swapped to a surviving peer if one exists,
// otherwise it is removed from WireGuard.
func (rm *AllowedIPsRefCounter) Decrement(prefix netip.Prefix, peerKey string) (Ref[string], error) {
rm.mu.Lock()
defer rm.mu.Unlock()
e, ok := rm.entries[prefix]
if !ok {
logCallerF("No allowed IP reference found for prefix %v", prefix)
return Ref[string]{}, nil
}
if e.peers[peerKey] > 0 {
logCallerF("Decreasing allowed IP ref count for prefix %v peer %s [peer %d -> %d, total %d -> %d, active %q]",
prefix, peerKey, e.peers[peerKey], e.peers[peerKey]-1, e.total, e.total-1, e.active)
e.peers[peerKey]--
e.total--
if e.peers[peerKey] == 0 {
delete(e.peers, peerKey)
}
} else {
logCallerF("No allowed IP reference found for prefix %v peer %s", prefix, peerKey)
}
// If the peer currently installed in WireGuard still holds references, nothing to reprogram.
// Keying the check on the active peer (not the one just released) makes this self-healing:
// a prior swap whose remove/add failed leaves e.active pointing at a peer with no references,
// and this retries the hand-off on the next Decrement instead of getting stuck.
if e.active != "" && e.peers[e.active] > 0 {
return Ref[string]{Count: e.total, Out: e.active}, nil
}
// Detach the stale/gone active peer from WireGuard before reprogramming.
if e.active != "" {
if err := rm.remove(prefix, e.active); err != nil {
return Ref[string]{Count: e.total, Out: e.active}, fmt.Errorf("remove allowed IP %v for peer %s: %w", prefix, e.active, err)
}
e.active = ""
}
// Hand the prefix over to a surviving peer, or drop the entry when none remain.
if survivor, ok := pickSurvivor(e.peers); ok {
out, err := rm.add(prefix, survivor)
if err != nil {
return Ref[string]{Count: e.total, Out: ""}, fmt.Errorf("swap allowed IP %v to peer %s: %w", prefix, survivor, err)
}
e.active = out
return Ref[string]{Count: e.total, Out: e.active}, nil
}
delete(rm.entries, prefix)
return Ref[string]{Count: 0, Out: ""}, nil
}
// Flush removes all prefixes from WireGuard and clears the counter.
func (rm *AllowedIPsRefCounter) Flush() error {
rm.mu.Lock()
defer rm.mu.Unlock()
var merr *multierror.Error
for prefix, e := range rm.entries {
if e.active == "" {
continue
}
logCallerF("Flushing allowed IP for prefix %v peer %s", prefix, e.active)
if err := rm.remove(prefix, e.active); err != nil {
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %v for peer %s: %w", prefix, e.active, err))
}
}
clear(rm.entries)
return nberrors.FormatErrorOrNil(merr)
}
// pickSurvivor deterministically selects a peer still referencing the prefix. WireGuard cannot do
// multipath for a single prefix, so any surviving peer is a valid winner; the choice is made stable
// (lowest peerKey) for predictable behavior and testability.
func pickSurvivor(peers map[string]int) (string, bool) {
if len(peers) == 0 {
return "", false
}
keys := make([]string, 0, len(peers))
for k := range peers {
keys = append(keys, k)
}
sort.Strings(keys)
return keys[0], true
}

View File

@@ -0,0 +1,241 @@
package refcounter
import (
"errors"
"net/netip"
"testing"
)
// fakeWG models WireGuard's cryptokey routing: a prefix can be installed on exactly one peer.
// failAdd/failRemove make the next add/remove fail once, to exercise the self-healing error paths.
type fakeWG struct {
installed map[netip.Prefix]string
adds int
removes int
failAdd bool
failRemove bool
}
func newFakeWG() *fakeWG {
return &fakeWG{installed: map[netip.Prefix]string{}}
}
func (f *fakeWG) counter() *AllowedIPsRefCounter {
return NewAllowedIPs(
func(prefix netip.Prefix, peerKey string) (string, error) {
if f.failAdd {
f.failAdd = false
return "", errors.New("add failed")
}
f.adds++
f.installed[prefix] = peerKey
return peerKey, nil
},
func(prefix netip.Prefix, peerKey string) error {
if f.failRemove {
f.failRemove = false
return errors.New("remove failed")
}
f.removes++
// only clear if this peer is the one installed, mirroring wg semantics
if f.installed[prefix] == peerKey {
delete(f.installed, prefix)
}
return nil
},
)
}
func mustPrefix(t *testing.T, s string) netip.Prefix {
t.Helper()
p, err := netip.ParsePrefix(s)
if err != nil {
t.Fatalf("parse prefix %q: %v", s, err)
}
return p
}
func mustIncrement(t *testing.T, c *AllowedIPsRefCounter, p netip.Prefix, peer string) Ref[string] {
t.Helper()
ref, err := c.Increment(p, peer)
if err != nil {
t.Fatalf("Increment(%v, %s): %v", p, peer, err)
}
return ref
}
func mustDecrement(t *testing.T, c *AllowedIPsRefCounter, p netip.Prefix, peer string) Ref[string] {
t.Helper()
ref, err := c.Decrement(p, peer)
if err != nil {
t.Fatalf("Decrement(%v, %s): %v", p, peer, err)
}
return ref
}
// TestAllowedIPs_SwapOnActivePeerRemoval reproduces the reported bug: two networks with the same
// prefix routed by different peers. Removing the network whose peer is installed must hand the
// prefix over to the surviving peer instead of leaving it on the removed one.
func TestAllowedIPs_SwapOnActivePeerRemoval(t *testing.T) {
f := newFakeWG()
c := f.counter()
p := mustPrefix(t, "10.44.8.0/24")
mustIncrement(t, c, p, "peerA")
mustIncrement(t, c, p, "peerB")
// First peer wins while both are present.
if got := f.installed[p]; got != "peerA" {
t.Fatalf("expected peerA installed, got %q", got)
}
// Remove the active peer's network -> must swap to peerB.
mustDecrement(t, c, p, "peerA")
if got := f.installed[p]; got != "peerB" {
t.Fatalf("BUG: prefix stuck on removed peer, want peerB got %q", got)
}
// Remove the last one -> prefix gone.
mustDecrement(t, c, p, "peerB")
if _, ok := f.installed[p]; ok {
t.Fatalf("expected prefix removed, still installed on %q", f.installed[p])
}
}
// TestAllowedIPs_RemoveNonActivePeer removing a non-installed peer must not touch WireGuard.
func TestAllowedIPs_RemoveNonActivePeer(t *testing.T) {
f := newFakeWG()
c := f.counter()
p := mustPrefix(t, "10.44.8.0/24")
mustIncrement(t, c, p, "peerA")
mustIncrement(t, c, p, "peerB")
removesBefore := f.removes
mustDecrement(t, c, p, "peerB")
if f.installed[p] != "peerA" {
t.Fatalf("active peer must stay peerA, got %q", f.installed[p])
}
if f.removes != removesBefore {
t.Fatalf("removing a non-active peer must not call wg remove")
}
}
// TestAllowedIPs_SamePeerMultipleRefs two references via the same peer must keep the prefix until
// the last reference is released (the reason the per-peer count must be an int, not a set).
func TestAllowedIPs_SamePeerMultipleRefs(t *testing.T) {
f := newFakeWG()
c := f.counter()
p := mustPrefix(t, "10.44.8.0/24")
mustIncrement(t, c, p, "peerA")
mustIncrement(t, c, p, "peerA")
if f.adds != 1 {
t.Fatalf("expected a single wg add for the same peer, got %d", f.adds)
}
mustDecrement(t, c, p, "peerA")
if f.installed[p] != "peerA" {
t.Fatalf("prefix must stay while a reference remains, got %q", f.installed[p])
}
if f.removes != 0 {
t.Fatalf("no wg remove expected while a reference remains, got %d", f.removes)
}
mustDecrement(t, c, p, "peerA")
if _, ok := f.installed[p]; ok {
t.Fatalf("prefix must be removed after last reference")
}
}
// TestAllowedIPs_RefCountAndActive checks the Ref returned to callers (used for the HA-disabled log).
func TestAllowedIPs_RefCountAndActive(t *testing.T) {
f := newFakeWG()
c := f.counter()
p := mustPrefix(t, "10.44.8.0/24")
ref := mustIncrement(t, c, p, "peerA")
if ref.Count != 1 || ref.Out != "peerA" {
t.Fatalf("want {1, peerA}, got {%d, %q}", ref.Count, ref.Out)
}
ref = mustIncrement(t, c, p, "peerB")
if ref.Count != 2 || ref.Out != "peerA" {
t.Fatalf("want {2, peerA}, got {%d, %q}", ref.Count, ref.Out)
}
}
// TestAllowedIPs_Flush removes everything installed and clears the counter.
func TestAllowedIPs_Flush(t *testing.T) {
f := newFakeWG()
c := f.counter()
p1 := mustPrefix(t, "10.44.8.0/24")
p2 := mustPrefix(t, "10.44.9.0/24")
mustIncrement(t, c, p1, "peerA")
mustIncrement(t, c, p2, "peerB")
if err := c.Flush(); err != nil {
t.Fatal(err)
}
if len(f.installed) != 0 {
t.Fatalf("expected all prefixes removed, got %v", f.installed)
}
// After flush, a fresh increment must add again.
mustIncrement(t, c, p1, "peerC")
if f.installed[p1] != "peerC" {
t.Fatalf("counter not reset after flush")
}
}
// TestAllowedIPs_SelfHealAfterSwapAddError ensures a failed add during a swap does not permanently
// strand the prefix: the next Decrement (or Increment) must retry and install a surviving peer.
func TestAllowedIPs_SelfHealAfterSwapAddError(t *testing.T) {
f := newFakeWG()
c := f.counter()
p := mustPrefix(t, "10.44.8.0/24")
mustIncrement(t, c, p, "peerA")
mustIncrement(t, c, p, "peerB")
mustIncrement(t, c, p, "peerC")
// Removing the active peerA triggers a swap to a survivor; make the add fail once.
f.failAdd = true
if _, err := c.Decrement(p, "peerA"); err == nil {
t.Fatalf("expected error from failed swap add")
}
if _, ok := f.installed[p]; ok {
t.Fatalf("nothing should be installed after a failed swap add, got %q", f.installed[p])
}
// A later Decrement of a non-active survivor must retry the hand-off (self-heal), not stay stuck.
ref := mustDecrement(t, c, p, "peerC")
if got := f.installed[p]; got == "" {
t.Fatalf("self-heal failed: prefix left unrouted after add recovered")
}
if ref.Out == "" {
t.Fatalf("expected an active peer after self-heal, got empty")
}
}
// TestAllowedIPs_SelfHealAfterRemoveError ensures a failed remove during a swap is retried instead
// of leaving e.active stuck on a peer that no longer holds references.
func TestAllowedIPs_SelfHealAfterRemoveError(t *testing.T) {
f := newFakeWG()
c := f.counter()
p := mustPrefix(t, "10.44.8.0/24")
mustIncrement(t, c, p, "peerA")
mustIncrement(t, c, p, "peerB")
// Releasing active peerA must detach it (remove) then add peerB; fail the remove once.
f.failRemove = true
if _, err := c.Decrement(p, "peerA"); err == nil {
t.Fatalf("expected error from failed remove")
}
// Next Decrement of the non-active survivor retries: removes stale peerA, installs peerB.
mustDecrement(t, c, p, "peerB")
// peerB had only one ref, so after retry the prefix is fully released.
if _, ok := f.installed[p]; ok {
t.Fatalf("expected prefix released after self-heal, still on %q", f.installed[p])
}
}

View File

@@ -5,5 +5,7 @@ import "net/netip"
// RouteRefCounter is a Counter for Route, it doesn't take any input on Increment and doesn't use any output on Decrement
type RouteRefCounter = Counter[netip.Prefix, struct{}, struct{}]
// AllowedIPsRefCounter is a Counter for AllowedIPs, it takes a peer key on Increment and passes it back to Decrement
type AllowedIPsRefCounter = Counter[netip.Prefix, string, string]
// AllowedIPsRefCounter tracks WireGuard AllowedIPs per prefix. Unlike the generic Counter it is peer-aware:
// a prefix can be claimed by several peers at once and WireGuard allows a given prefix on exactly one peer,
// so the counter records the per-peer reference count and swaps the installed peer when the active one is released.
// See allowedips.go.

View File

@@ -15,6 +15,11 @@ type Route struct {
route *route.Route
routeRefCounter *refcounter.RouteRefCounter
allowedIPsRefcounter *refcounter.AllowedIPsRefCounter
// currentPeerKey is the routing peer this watcher currently has the prefix installed on
// (the HA winner elected by the watcher). It can differ from route.Peer and change on
// failover, so it is recorded on AddAllowedIPs and used on RemoveAllowedIPs to decrement
// the exact peer that was incremented.
currentPeerKey string
}
func NewRoute(params common.HandlerParams) *Route {
@@ -52,12 +57,15 @@ func (r *Route) AddAllowedIPs(peerKey string) error {
ref.Out,
)
}
r.currentPeerKey = peerKey
return nil
}
func (r *Route) RemoveAllowedIPs() error {
if _, err := r.allowedIPsRefcounter.Decrement(r.route.Network); err != nil {
return err
var err error
if _, decErr := r.allowedIPsRefcounter.Decrement(r.route.Network, r.currentPeerKey); decErr != nil {
err = fmt.Errorf("remove allowed IP %s: %w", r.route.Network, decErr)
}
return nil
r.currentPeerKey = ""
return err
}

View File

@@ -44,25 +44,10 @@ type Auth struct {
// NewAuth instantiate Auth struct and validate the management URL
func NewAuth(cfgPath string, mgmURL string) (*Auth, error) {
inputCfg := profilemanager.ConfigInput{
ConfigPath: cfgPath,
ManagementURL: mgmURL,
}
// Load the existing config when a config file is already present so an
// interactive re-login reuses the peer's persisted WireGuard private key
// (and thus its identity) instead of generating a fresh one. Generating a
// new key registers a brand-new peer on the management server on every
// re-auth (named after the fallback hostname). Only fall back to a fresh
// in-memory config for the first-time login when no config file exists yet.
// DirectUpdateOrCreateConfig uses non-atomic writes so it also works inside
// the tvOS App Group sandbox where atomic temp-file+rename is blocked.
var cfg *profilemanager.Config
var err error
if cfgPath != "" {
cfg, err = profilemanager.DirectUpdateOrCreateConfig(inputCfg)
} else {
cfg, err = profilemanager.CreateInMemoryConfig(inputCfg)
}
cfg, err := profilemanager.CreateInMemoryConfig(inputCfg)
if err != nil {
return nil, err
}

View File

@@ -1,27 +1,71 @@
#!/usr/bin/env bash
# Runs the NetBird daemon and brings the connection up in one container process.
#
# A thin wrapper is needed (rather than a one-line ENTRYPOINT) for two reasons:
# 1. Two processes must run: the daemon (`service run`, long-lived) and a
# one-shot `up` that brings the connection up.
# 2. Signal handling: as PID 1 the wrapper must forward SIGTERM/SIGINT to the
# daemon so it tears down WireGuard and deregisters ephemeral peers on
# `docker stop`. Without this the daemon would be killed uncleanly.
#
# `netbird up` waits for the daemon to become ready on its own, so no readiness
# poll is needed here.
set -eEuo pipefail
: ${NB_ENTRYPOINT_SERVICE_TIMEOUT:="30"}
NETBIRD_BIN="${NETBIRD_BIN:-"netbird"}"
export NB_LOG_FILE="${NB_LOG_FILE:-"console,/var/log/netbird/client.log"}"
service_pids=()
daemon=""
cleanup() { [[ -n "${daemon}" ]] && kill -TERM "${daemon}" 2>/dev/null || true; }
trap cleanup SIGTERM SIGINT EXIT
_log() {
# mimic Go logger's output for easier parsing
# 2025-04-15T21:32:00+08:00 INFO client/internal/config.go:495: setting notifications to disabled by default
printf "$(date -Isec) ${1} ${BASH_SOURCE[1]}:${BASH_LINENO[1]}: ${2}\n" "${@:3}" >&2
}
"${NETBIRD_BIN}" service run &
daemon=$!
info() {
_log INFO "$@"
}
"${NETBIRD_BIN}" up
warn() {
_log WARN "$@"
}
wait "${daemon}"
on_exit() {
info "Shutting down NetBird daemon..."
if test "${#service_pids[@]}" -gt 0; then
info "terminating service process IDs: ${service_pids[@]@Q}"
kill -TERM "${service_pids[@]}" 2>/dev/null || true
wait "${service_pids[@]}" 2>/dev/null || true
else
info "there are no service processes to terminate"
fi
}
wait_for_daemon_startup() {
local timeout="${1}"
if [[ "${timeout}" -eq 0 ]]; then
info "not waiting for daemon startup due to zero timeout."
return
fi
local deadline=$((SECONDS + timeout))
while [[ "${SECONDS}" -lt "${deadline}" ]]; do
if "${NETBIRD_BIN}" status --check live 2>/dev/null; then
return
fi
sleep 1
done
warn "daemon did not become responsive after ${timeout} seconds, exiting..."
exit 1
}
connect() {
info "running 'netbird up'..."
"${NETBIRD_BIN}" up
return $?
}
main() {
trap 'on_exit' SIGTERM SIGINT EXIT
"${NETBIRD_BIN}" service run &
service_pids+=("$!")
info "registered new service process 'netbird service run', currently running: ${service_pids[@]@Q}"
wait_for_daemon_startup "${NB_ENTRYPOINT_SERVICE_TIMEOUT}"
connect
wait "${service_pids[@]}"
}
main "$@"

View File

@@ -995,13 +995,8 @@ type StatusResponse struct {
// Unset when the peer is not SSO-registered or login expiration is disabled.
// The UI derives "warning active" from this value and its own clock.
SessionExpiresAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=sessionExpiresAt,proto3" json:"sessionExpiresAt,omitempty"`
// daemonReady reports whether the daemon has finished starting up and is
// serving RPCs backed by a running engine. Older daemons never set this
// (it defaults to false); clients must treat an unset value as "unknown"
// and fall back to their previous readiness heuristics for compatibility.
DaemonReady bool `protobuf:"varint,5,opt,name=daemonReady,proto3" json:"daemonReady,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *StatusResponse) Reset() {
@@ -1062,13 +1057,6 @@ func (x *StatusResponse) GetSessionExpiresAt() *timestamppb.Timestamp {
return nil
}
func (x *StatusResponse) GetDaemonReady() bool {
if x != nil {
return x.DaemonReady
}
return false
}
type DownRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
unknownFields protoimpl.UnknownFields
@@ -7095,15 +7083,14 @@ const file_daemon_proto_rawDesc = "" +
"\x11getFullPeerStatus\x18\x01 \x01(\bR\x11getFullPeerStatus\x12(\n" +
"\x0fshouldRunProbes\x18\x02 \x01(\bR\x0fshouldRunProbes\x12'\n" +
"\fwaitForReady\x18\x03 \x01(\bH\x00R\fwaitForReady\x88\x01\x01B\x0f\n" +
"\r_waitForReady\"\xec\x01\n" +
"\r_waitForReady\"\xca\x01\n" +
"\x0eStatusResponse\x12\x16\n" +
"\x06status\x18\x01 \x01(\tR\x06status\x122\n" +
"\n" +
"fullStatus\x18\x02 \x01(\v2\x12.daemon.FullStatusR\n" +
"fullStatus\x12$\n" +
"\rdaemonVersion\x18\x03 \x01(\tR\rdaemonVersion\x12F\n" +
"\x10sessionExpiresAt\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\x10sessionExpiresAt\x12 \n" +
"\vdaemonReady\x18\x05 \x01(\bR\vdaemonReady\"\r\n" +
"\x10sessionExpiresAt\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\x10sessionExpiresAt\"\r\n" +
"\vDownRequest\"\x0e\n" +
"\fDownResponse\"P\n" +
"\x10GetConfigRequest\x12 \n" +

View File

@@ -291,11 +291,6 @@ message StatusResponse{
// Unset when the peer is not SSO-registered or login expiration is disabled.
// The UI derives "warning active" from this value and its own clock.
google.protobuf.Timestamp sessionExpiresAt = 4;
// daemonReady reports whether the daemon has finished starting up and is
// serving RPCs backed by a running engine. Older daemons never set this
// (it defaults to false); clients must treat an unset value as "unknown"
// and fall back to their previous readiness heuristics for compatibility.
bool daemonReady = 5;
}
message DownRequest {}

View File

@@ -104,12 +104,6 @@ type Server struct {
persistSyncResponse bool
isSessionActive atomic.Bool
// ready is set once the daemon has finished startup and is serving RPCs
// backed by a running engine (see SetReady, called after Start succeeds and
// the service is registered). Reported via StatusResponse.DaemonReady so
// clients can wait deterministically instead of polling heuristically.
ready atomic.Bool
cpuProfileBuf *bytes.Buffer
cpuProfiling bool
@@ -164,14 +158,6 @@ func New(ctx context.Context, logFile string, configFile string, profilesDisable
return s
}
// SetReady marks the daemon as fully started and serving RPCs. It is called by
// the service controller once Start has succeeded and the DaemonService is
// registered, so a subsequent Status RPC reports DaemonReady=true. Safe for
// concurrent use.
func (s *Server) SetReady() {
s.ready.Store(true)
}
func (s *Server) Start() error {
s.mutex.Lock()
defer s.mutex.Unlock()
@@ -1436,7 +1422,7 @@ func (s *Server) buildStatusResponse(ctx context.Context, msg *proto.StatusReque
s.isSessionActive.Store(false)
}
statusResponse := proto.StatusResponse{Status: string(status), DaemonVersion: version.NetbirdVersion(), DaemonReady: s.ready.Load()}
statusResponse := proto.StatusResponse{Status: string(status), DaemonVersion: version.NetbirdVersion()}
if deadline := s.statusRecorder.GetSessionExpiresAt(); !deadline.IsZero() {
statusResponse.SessionExpiresAt = timestamppb.New(deadline)

View File

@@ -17,7 +17,8 @@ RUN apk add --no-cache bash ca-certificates ip6tables iproute2 iptables
ENV NETBIRD_BIN="/usr/local/bin/netbird" \
NB_LOG_FILE="console,/var/log/netbird/client.log" \
NB_DAEMON_ADDR="unix:///var/run/netbird.sock" \
NB_ENABLE_CAPTURE="false"
NB_ENABLE_CAPTURE="false" \
NB_ENTRYPOINT_SERVICE_TIMEOUT="30"
ENTRYPOINT [ "/usr/local/bin/netbird-entrypoint.sh" ]
COPY client/netbird-entrypoint.sh /usr/local/bin/netbird-entrypoint.sh
COPY --from=builder /out/netbird /usr/local/bin/netbird