mirror of
https://github.com/netbirdio/netbird.git
synced 2026-07-22 16:31:28 +02:00
Compare commits
2 Commits
fix/cli-up
...
fix/routes
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aa4257f1ac | ||
|
|
f6a756c962 |
@@ -17,7 +17,8 @@ ENV \
|
||||
NETBIRD_BIN="/usr/local/bin/netbird" \
|
||||
NB_LOG_FILE="console,/var/log/netbird/client.log" \
|
||||
NB_DAEMON_ADDR="unix:///var/run/netbird.sock" \
|
||||
NB_ENABLE_CAPTURE="false"
|
||||
NB_ENABLE_CAPTURE="false" \
|
||||
NB_ENTRYPOINT_SERVICE_TIMEOUT="30"
|
||||
|
||||
ENTRYPOINT [ "/usr/local/bin/netbird-entrypoint.sh" ]
|
||||
ARG TARGETPLATFORM
|
||||
|
||||
@@ -23,7 +23,8 @@ ENV \
|
||||
NB_DAEMON_ADDR="unix:///var/lib/netbird/netbird.sock" \
|
||||
NB_LOG_FILE="console,/var/lib/netbird/client.log" \
|
||||
NB_DISABLE_DNS="true" \
|
||||
NB_ENABLE_CAPTURE="false"
|
||||
NB_ENABLE_CAPTURE="false" \
|
||||
NB_ENTRYPOINT_SERVICE_TIMEOUT="30"
|
||||
|
||||
ENTRYPOINT [ "/usr/local/bin/netbird-entrypoint.sh" ]
|
||||
ARG TARGETPLATFORM
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/connectivity"
|
||||
|
||||
"github.com/netbirdio/netbird/client/internal"
|
||||
"github.com/netbirdio/netbird/client/proto"
|
||||
)
|
||||
|
||||
// startUnixGRPCServer starts a bare gRPC server listening on a unix socket at path
|
||||
// and returns a stop function. No services are registered; the connectivity-state
|
||||
// wait only cares about the transport becoming READY.
|
||||
func startUnixGRPCServer(t *testing.T, path string) func() {
|
||||
t.Helper()
|
||||
lis, err := net.Listen("unix", path)
|
||||
if err != nil {
|
||||
t.Fatalf("listen unix %s: %v", path, err)
|
||||
}
|
||||
srv := grpc.NewServer()
|
||||
go func() { _ = srv.Serve(lis) }()
|
||||
return srv.Stop
|
||||
}
|
||||
|
||||
func TestDialClientGRPCServer_ConnectsWhenServing(t *testing.T) {
|
||||
sock := filepath.Join(t.TempDir(), "nb.sock")
|
||||
stop := startUnixGRPCServer(t, sock)
|
||||
defer stop()
|
||||
|
||||
conn, err := dialClientGRPCServer(context.Background(), "unix://"+sock, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("expected connection, got error: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
if state := conn.GetState(); state != connectivity.Ready {
|
||||
t.Fatalf("expected READY, got %s", state)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDialClientGRPCServer_WaitsForLateServer is the core regression test: the
|
||||
// daemon socket appears only after the dial has already started, mirroring
|
||||
// "netbird service start" immediately followed by "netbird up".
|
||||
func TestDialClientGRPCServer_WaitsForLateServer(t *testing.T) {
|
||||
sock := filepath.Join(t.TempDir(), "nb.sock")
|
||||
|
||||
var stop func()
|
||||
timer := time.AfterFunc(1*time.Second, func() {
|
||||
stop = startUnixGRPCServer(t, sock)
|
||||
})
|
||||
defer timer.Stop()
|
||||
defer func() {
|
||||
if stop != nil {
|
||||
stop()
|
||||
}
|
||||
}()
|
||||
|
||||
start := time.Now()
|
||||
conn, err := dialClientGRPCServer(context.Background(), "unix://"+sock, 10*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("expected connection after late server start, got error: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
if elapsed := time.Since(start); elapsed < 500*time.Millisecond {
|
||||
t.Fatalf("connected too fast (%s); server should not have been up yet", elapsed)
|
||||
}
|
||||
if state := conn.GetState(); state != connectivity.Ready {
|
||||
t.Fatalf("expected READY, got %s", state)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeStatusServer serves the Status RPC with a programmable response so we can
|
||||
// exercise waitForDaemonStatus without spinning up a real engine.
|
||||
type fakeStatusServer struct {
|
||||
proto.UnimplementedDaemonServiceServer
|
||||
resp func() *proto.StatusResponse
|
||||
}
|
||||
|
||||
func (f *fakeStatusServer) Status(context.Context, *proto.StatusRequest) (*proto.StatusResponse, error) {
|
||||
return f.resp(), nil
|
||||
}
|
||||
|
||||
func startFakeStatusServer(t *testing.T, sock string, resp func() *proto.StatusResponse) func() {
|
||||
t.Helper()
|
||||
lis, err := net.Listen("unix", sock)
|
||||
if err != nil {
|
||||
t.Fatalf("listen unix %s: %v", sock, err)
|
||||
}
|
||||
srv := grpc.NewServer()
|
||||
proto.RegisterDaemonServiceServer(srv, &fakeStatusServer{resp: resp})
|
||||
go func() { _ = srv.Serve(lis) }()
|
||||
return srv.Stop
|
||||
}
|
||||
|
||||
func dialFake(t *testing.T, sock string) proto.DaemonServiceClient {
|
||||
t.Helper()
|
||||
conn, err := dialClientGRPCServer(context.Background(), "unix://"+sock, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("dial fake daemon: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { conn.Close() })
|
||||
return proto.NewDaemonServiceClient(conn)
|
||||
}
|
||||
|
||||
// New daemon that flips DaemonReady=true after a couple of polls: waitForDaemonStatus
|
||||
// must block until the flag is set, then return.
|
||||
func TestWaitForDaemonStatus_WaitsForDaemonReady(t *testing.T) {
|
||||
sock := filepath.Join(t.TempDir(), "nb.sock")
|
||||
var polls int32
|
||||
stop := startFakeStatusServer(t, sock, func() *proto.StatusResponse {
|
||||
n := atomic.AddInt32(&polls, 1)
|
||||
return &proto.StatusResponse{
|
||||
Status: string(internal.StatusConnecting),
|
||||
DaemonReady: n >= 3, // ready only from the 3rd poll on
|
||||
}
|
||||
})
|
||||
defer stop()
|
||||
|
||||
client := dialFake(t, sock)
|
||||
status, err := waitForDaemonStatus(context.Background(), client)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if !status.GetDaemonReady() {
|
||||
t.Fatalf("expected DaemonReady=true, got false")
|
||||
}
|
||||
if got := atomic.LoadInt32(&polls); got < 3 {
|
||||
t.Fatalf("expected at least 3 polls before ready, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Older daemon that never sets DaemonReady but reports a healthy (Connected)
|
||||
// status: waitForDaemonStatus must return promptly via the readiness fallback,
|
||||
// not block for the whole grace window.
|
||||
func TestWaitForDaemonStatus_OlderDaemonHealthyStatus(t *testing.T) {
|
||||
sock := filepath.Join(t.TempDir(), "nb.sock")
|
||||
stop := startFakeStatusServer(t, sock, func() *proto.StatusResponse {
|
||||
return &proto.StatusResponse{Status: string(internal.StatusConnected)} // DaemonReady unset
|
||||
})
|
||||
defer stop()
|
||||
|
||||
client := dialFake(t, sock)
|
||||
start := time.Now()
|
||||
status, err := waitForDaemonStatus(context.Background(), client)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if status.GetDaemonReady() {
|
||||
t.Fatalf("expected DaemonReady=false from older daemon")
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > 2*time.Second {
|
||||
t.Fatalf("returned too slowly (%s); healthy status should short-circuit the grace", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDialClientGRPCServer_TimesOutWhenAbsent(t *testing.T) {
|
||||
sock := filepath.Join(t.TempDir(), "never.sock")
|
||||
|
||||
start := time.Now()
|
||||
conn, err := dialClientGRPCServer(context.Background(), "unix://"+sock, 1*time.Second)
|
||||
if err == nil {
|
||||
conn.Close()
|
||||
t.Fatal("expected timeout error, got nil")
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed < 900*time.Millisecond {
|
||||
t.Fatalf("returned too early (%s); should have waited ~timeout", elapsed)
|
||||
}
|
||||
}
|
||||
@@ -20,8 +20,6 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
"google.golang.org/grpc"
|
||||
gbackoff "google.golang.org/grpc/backoff"
|
||||
"google.golang.org/grpc/connectivity"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
|
||||
daddr "github.com/netbirdio/netbird/client/internal/daemonaddr"
|
||||
@@ -266,70 +264,17 @@ func FlagNameToEnvVar(cmdFlag string, prefix string) string {
|
||||
return prefix + upper
|
||||
}
|
||||
|
||||
// defaultDaemonDialTimeout is how long DialClientGRPCServer waits for the daemon
|
||||
// to become reachable. It is intentionally generous so that invoking the CLI
|
||||
// right after "netbird service start" (e.g. from a container entrypoint) tolerates
|
||||
// the window where the daemon has created its socket but is not yet serving.
|
||||
const defaultDaemonDialTimeout = 30 * time.Second
|
||||
|
||||
// DialClientGRPCServer returns a client connection to the daemon server. It waits
|
||||
// for the daemon to become reachable, retrying with backoff until the connection
|
||||
// reports READY or defaultDaemonDialTimeout elapses. This handles the startup race
|
||||
// where the daemon socket exists (or is about to) before the gRPC server is serving.
|
||||
// DialClientGRPCServer returns client connection to the daemon server.
|
||||
func DialClientGRPCServer(ctx context.Context, addr string) (*grpc.ClientConn, error) {
|
||||
return dialClientGRPCServer(ctx, addr, defaultDaemonDialTimeout)
|
||||
}
|
||||
|
||||
func dialClientGRPCServer(ctx context.Context, addr string, timeout time.Duration) (*grpc.ClientConn, error) {
|
||||
conn, err := grpc.NewClient(
|
||||
strings.TrimPrefix(addr, "tcp://"),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
// Cap reconnect backoff at 5s; gRPC's default 120s MaxDelay would leave the
|
||||
// CLI waiting far too long to notice a freshly-started daemon. Mirrors the GUI.
|
||||
grpc.WithConnectParams(grpc.ConnectParams{
|
||||
Backoff: gbackoff.Config{
|
||||
BaseDelay: 1 * time.Second,
|
||||
Multiplier: 1.6,
|
||||
Jitter: 0.2,
|
||||
MaxDelay: 5 * time.Second,
|
||||
},
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create daemon gRPC client: %w", err)
|
||||
}
|
||||
|
||||
// grpc.NewClient is lazy: it does not connect until the first RPC or until we
|
||||
// nudge it. Trigger connection attempts and wait until the channel reaches READY.
|
||||
if err := waitForConnReady(ctx, conn, timeout); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
// waitForConnReady drives the gRPC channel out of IDLE and blocks until it becomes
|
||||
// READY, or until timeout/ctx expires. TRANSIENT_FAILURE (daemon not yet serving)
|
||||
// is treated as retryable so the caller keeps waiting within the deadline.
|
||||
func waitForConnReady(ctx context.Context, conn *grpc.ClientConn, timeout time.Duration) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Second*10)
|
||||
defer cancel()
|
||||
|
||||
for {
|
||||
state := conn.GetState()
|
||||
switch state {
|
||||
case connectivity.Ready:
|
||||
return nil
|
||||
case connectivity.Idle:
|
||||
// Kick the lazy channel into connecting.
|
||||
conn.Connect()
|
||||
}
|
||||
|
||||
if !conn.WaitForStateChange(ctx, state) {
|
||||
// ctx expired while in `state`.
|
||||
return fmt.Errorf("timed out after %s waiting for daemon to become ready (last state: %s)", timeout, state)
|
||||
}
|
||||
}
|
||||
return grpc.DialContext(
|
||||
ctx,
|
||||
strings.TrimPrefix(addr, "tcp://"),
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithBlock(),
|
||||
)
|
||||
}
|
||||
|
||||
// WithBackOff execute function in backoff cycle.
|
||||
|
||||
@@ -78,10 +78,6 @@ func (p *program) Start(svc service.Service) error {
|
||||
log.Fatalf("failed to start daemon: %v", err)
|
||||
}
|
||||
proto.RegisterDaemonServiceServer(p.serv, serverInstance)
|
||||
// The engine is started and the service is registered: from here on the
|
||||
// daemon serves RPCs backed by a running engine. Report readiness so
|
||||
// clients (e.g. netbird up) can wait deterministically.
|
||||
serverInstance.SetReady()
|
||||
|
||||
p.serverInstanceMu.Lock()
|
||||
p.serverInstance = serverInstance
|
||||
|
||||
@@ -295,7 +295,9 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command, pm *profilemanager
|
||||
|
||||
client := proto.NewDaemonServiceClient(conn)
|
||||
|
||||
status, err := waitForDaemonStatus(ctx, client)
|
||||
status, err := client.Status(ctx, &proto.StatusRequest{
|
||||
WaitForReady: func() *bool { b := true; return &b }(),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to get daemon status: %v", err)
|
||||
}
|
||||
@@ -334,79 +336,6 @@ func runInDaemonMode(ctx context.Context, cmd *cobra.Command, pm *profilemanager
|
||||
return nil
|
||||
}
|
||||
|
||||
// daemonStatusPollTimeout bounds how long we poll the Status RPC waiting for the
|
||||
// daemon to answer coherently. The transport is already READY at this point (see
|
||||
// DialClientGRPCServer), so this only covers the brief window where the gRPC server
|
||||
// is serving but the daemon engine is still starting up and the Status RPC races
|
||||
// against server.Start().
|
||||
const daemonStatusPollTimeout = 15 * time.Second
|
||||
|
||||
// daemonReadyGrace bounds how long we keep polling once the daemon answers but
|
||||
// still reports DaemonReady=false. A freshly-started daemon flips it to true
|
||||
// within this window; an older daemon that never sets the field simply falls
|
||||
// through after the grace elapses, preserving backward compatibility.
|
||||
const daemonReadyGrace = 10 * time.Second
|
||||
|
||||
// waitForDaemonStatus fetches the daemon status, waiting for the daemon to become
|
||||
// ready. It handles two startup races:
|
||||
//
|
||||
// 1. The gRPC server is not yet serving: Status fails with Unavailable — retry.
|
||||
// 2. The server serves but the engine is still starting: a DaemonReady-aware
|
||||
// daemon reports DaemonReady=false until Start finishes; poll until it flips
|
||||
// true (bounded by daemonReadyGrace). Older daemons never set DaemonReady, so
|
||||
// we stop waiting on it after the grace and use the status as-is.
|
||||
//
|
||||
// It gives up after daemonStatusPollTimeout.
|
||||
func waitForDaemonStatus(ctx context.Context, client proto.DaemonServiceClient) (*proto.StatusResponse, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, daemonStatusPollTimeout)
|
||||
defer cancel()
|
||||
|
||||
waitForReady := true
|
||||
req := &proto.StatusRequest{WaitForReady: &waitForReady}
|
||||
|
||||
var lastErr error
|
||||
var firstAnswer time.Time
|
||||
for {
|
||||
status, err := client.Status(ctx, req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
// Only retry while the daemon is not yet answering; surface real errors.
|
||||
if s, ok := gstatus.FromError(err); !ok || s.Code() != codes.Unavailable {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
// Daemon answered. Explicitly ready (DaemonReady-aware daemon), or
|
||||
// already fully connected — either way, done. Connected is the only
|
||||
// status unambiguous enough to short-circuit on: a DaemonReady-aware
|
||||
// daemon sets the flag at startup, so trusting Connected here can only
|
||||
// help an older daemon that never sets the flag, without overriding a
|
||||
// new daemon that legitimately reports DaemonReady=false while starting.
|
||||
if status.GetDaemonReady() || internal.StatusType(status.GetStatus()) == internal.StatusConnected {
|
||||
return status, nil
|
||||
}
|
||||
// Answered but neither ready-flagged nor connected yet: give a
|
||||
// DaemonReady-aware daemon a bounded window to finish starting, then
|
||||
// fall through so an older daemon that never sets the flag isn't
|
||||
// blocked here.
|
||||
if firstAnswer.IsZero() {
|
||||
firstAnswer = time.Now()
|
||||
} else if time.Since(firstAnswer) >= daemonReadyGrace {
|
||||
return status, nil
|
||||
}
|
||||
lastErr = nil
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if lastErr != nil {
|
||||
return nil, fmt.Errorf("daemon did not become ready: %w", lastErr)
|
||||
}
|
||||
return nil, fmt.Errorf("daemon did not become ready within %s", daemonStatusPollTimeout)
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func doDaemonUp(ctx context.Context, cmd *cobra.Command, client proto.DaemonServiceClient, pm *profilemanager.ProfileManager, activeProf *profilemanager.Profile, customDNSAddressConverted []byte, username string) error {
|
||||
|
||||
providedSetupKey, err := getSetupKey()
|
||||
|
||||
@@ -292,16 +292,18 @@ func (s *serviceViaListener) generateFreePort() (uint16, error) {
|
||||
return customPort, nil
|
||||
}
|
||||
|
||||
probeListener, err := net.ListenUDP("udp4", &net.UDPAddr{})
|
||||
udpAddr := net.UDPAddrFromAddrPort(netip.MustParseAddrPort("0.0.0.0:0"))
|
||||
probeListener, err := net.ListenUDP("udp", udpAddr)
|
||||
if err != nil {
|
||||
log.Debugf("failed to bind random port for DNS: %s", err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
port := uint16(probeListener.LocalAddr().(*net.UDPAddr).Port)
|
||||
if err = probeListener.Close(); err != nil {
|
||||
addrPort := netip.MustParseAddrPort(probeListener.LocalAddr().String()) // might panic if address is incorrect
|
||||
err = probeListener.Close()
|
||||
if err != nil {
|
||||
log.Debugf("failed to free up DNS port: %s", err)
|
||||
return 0, err
|
||||
}
|
||||
return port, nil
|
||||
return addrPort.Port(), nil
|
||||
}
|
||||
|
||||
@@ -95,7 +95,7 @@ func (d *DnsInterceptor) RemoveRoute() error {
|
||||
|
||||
// AllowedIPs should use real IPs
|
||||
if d.currentPeerKey != "" {
|
||||
if _, err := d.allowedIPsRefcounter.Decrement(prefix); err != nil {
|
||||
if _, err := d.allowedIPsRefcounter.Decrement(prefix, d.currentPeerKey); err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %v", prefix, err))
|
||||
}
|
||||
}
|
||||
@@ -172,7 +172,7 @@ func (d *DnsInterceptor) removeAllowedIP(realPrefix netip.Prefix) error {
|
||||
}
|
||||
|
||||
// AllowedIPs use real IPs
|
||||
if _, err := d.allowedIPsRefcounter.Decrement(realPrefix); err != nil {
|
||||
if _, err := d.allowedIPsRefcounter.Decrement(realPrefix, d.currentPeerKey); err != nil {
|
||||
return fmt.Errorf("remove allowed IP %s: %v", realPrefix, err)
|
||||
}
|
||||
|
||||
@@ -205,7 +205,7 @@ func (d *DnsInterceptor) RemoveAllowedIPs() error {
|
||||
for _, prefixes := range d.interceptedDomains {
|
||||
for _, prefix := range prefixes {
|
||||
// AllowedIPs use real IPs
|
||||
if _, err := d.allowedIPsRefcounter.Decrement(prefix); err != nil {
|
||||
if _, err := d.allowedIPsRefcounter.Decrement(prefix, d.currentPeerKey); err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %v", prefix, err))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,7 +135,7 @@ func (r *Route) RemoveAllowedIPs() error {
|
||||
var merr *multierror.Error
|
||||
for _, domainPrefixes := range r.dynamicDomains {
|
||||
for _, prefix := range domainPrefixes {
|
||||
if _, err := r.allowedIPsRefcounter.Decrement(prefix); err != nil {
|
||||
if _, err := r.allowedIPsRefcounter.Decrement(prefix, r.currentPeerKey); err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %w", prefix, err))
|
||||
}
|
||||
}
|
||||
@@ -320,7 +320,7 @@ func (r *Route) removeRoutes(prefixes []netip.Prefix) ([]netip.Prefix, error) {
|
||||
merr = multierror.Append(merr, fmt.Errorf("remove dynamic route for IP %s: %w", prefix, err))
|
||||
}
|
||||
if r.currentPeerKey != "" {
|
||||
if _, err := r.allowedIPsRefcounter.Decrement(prefix); err != nil {
|
||||
if _, err := r.allowedIPsRefcounter.Decrement(prefix, r.currentPeerKey); err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %s: %w", prefix, err))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -215,7 +215,7 @@ func (m *DefaultManager) setupRefCounters(useNoop bool) {
|
||||
)
|
||||
}
|
||||
|
||||
m.allowedIPsRefCounter = refcounter.New(
|
||||
m.allowedIPsRefCounter = refcounter.NewAllowedIPs(
|
||||
func(prefix netip.Prefix, peerKey string) (string, error) {
|
||||
// save peerKey to use it in the remove function
|
||||
return peerKey, m.wgInterface.AddAllowedIP(peerKey, prefix)
|
||||
|
||||
185
client/internal/routemanager/refcounter/allowedips.go
Normal file
185
client/internal/routemanager/refcounter/allowedips.go
Normal file
@@ -0,0 +1,185 @@
|
||||
package refcounter
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
|
||||
nberrors "github.com/netbirdio/netbird/client/errors"
|
||||
)
|
||||
|
||||
// allowedIPsEntry holds the per-peer reference counts for a single prefix and which peer is
|
||||
// currently installed in WireGuard. WireGuard allows a prefix on exactly one peer, so at most
|
||||
// one peer is active at a time even when several peers reference the prefix.
|
||||
type allowedIPsEntry struct {
|
||||
// peers maps a peerKey to the number of references holding the prefix for that peer.
|
||||
peers map[string]int
|
||||
// active is the peerKey currently installed in WireGuard for this prefix ("" if none).
|
||||
active string
|
||||
// total is the sum of all per-peer reference counts (kept in sync with peers).
|
||||
total int
|
||||
}
|
||||
|
||||
// AllowedIPsRefCounter is a peer-aware reference counter for WireGuard AllowedIPs.
|
||||
//
|
||||
// The generic Counter keys only by prefix and remembers a single Out value set by the first
|
||||
// caller, which it never changes. That is wrong for AllowedIPs: two independent watchers (or
|
||||
// multiple resolved domains) can reference the same prefix through different peers, and when the
|
||||
// peer currently installed in WireGuard releases its last reference the prefix must be handed over
|
||||
// to a surviving peer instead of being left pointing at the released one.
|
||||
//
|
||||
// It calls add/remove (which program WireGuard) only on the transitions that matter:
|
||||
// - add on the first reference for a prefix, or when swapping the active peer;
|
||||
// - remove on the last reference for a prefix, or on the old peer during a swap.
|
||||
type AllowedIPsRefCounter struct {
|
||||
mu sync.Mutex
|
||||
entries map[netip.Prefix]*allowedIPsEntry
|
||||
add AddFunc[netip.Prefix, string, string]
|
||||
remove RemoveFunc[netip.Prefix, string]
|
||||
}
|
||||
|
||||
// NewAllowedIPs creates a new peer-aware AllowedIPs reference counter.
|
||||
// add programs a prefix on a peer in WireGuard and returns the peerKey to store as the active peer.
|
||||
// remove unprograms the prefix from the given peer.
|
||||
func NewAllowedIPs(add AddFunc[netip.Prefix, string, string], remove RemoveFunc[netip.Prefix, string]) *AllowedIPsRefCounter {
|
||||
return &AllowedIPsRefCounter{
|
||||
entries: map[netip.Prefix]*allowedIPsEntry{},
|
||||
add: add,
|
||||
remove: remove,
|
||||
}
|
||||
}
|
||||
|
||||
// Increment adds a reference to prefix for peerKey. WireGuard is programmed only for the first
|
||||
// reference to a prefix; while a different peer is already installed the prefix is left with it
|
||||
// (first peer wins, HA at the WireGuard layer is not possible) and only the reference count is kept.
|
||||
func (rm *AllowedIPsRefCounter) Increment(prefix netip.Prefix, peerKey string) (Ref[string], error) {
|
||||
rm.mu.Lock()
|
||||
defer rm.mu.Unlock()
|
||||
|
||||
e, ok := rm.entries[prefix]
|
||||
if !ok {
|
||||
e = &allowedIPsEntry{peers: map[string]int{}}
|
||||
rm.entries[prefix] = e
|
||||
}
|
||||
|
||||
logCallerF("Increasing allowed IP ref count for prefix %v peer %s [peer %d -> %d, total %d -> %d, active %q]",
|
||||
prefix, peerKey, e.peers[peerKey], e.peers[peerKey]+1, e.total, e.total+1, e.active)
|
||||
|
||||
// Program WireGuard only when nothing is installed yet for this prefix.
|
||||
if e.active == "" {
|
||||
out, err := rm.add(prefix, peerKey)
|
||||
if errors.Is(err, ErrIgnore) {
|
||||
if e.total == 0 {
|
||||
delete(rm.entries, prefix)
|
||||
}
|
||||
return Ref[string]{Count: e.total, Out: e.active}, nil
|
||||
}
|
||||
if err != nil {
|
||||
if e.total == 0 {
|
||||
delete(rm.entries, prefix)
|
||||
}
|
||||
return Ref[string]{}, fmt.Errorf("failed to add allowed IP %v for peer %s: %w", prefix, peerKey, err)
|
||||
}
|
||||
e.active = out
|
||||
}
|
||||
|
||||
e.peers[peerKey]++
|
||||
e.total++
|
||||
|
||||
return Ref[string]{Count: e.total, Out: e.active}, nil
|
||||
}
|
||||
|
||||
// Decrement removes a reference to prefix for peerKey. When the peer currently installed in
|
||||
// WireGuard releases its last reference, the prefix is swapped to a surviving peer if one exists,
|
||||
// otherwise it is removed from WireGuard.
|
||||
func (rm *AllowedIPsRefCounter) Decrement(prefix netip.Prefix, peerKey string) (Ref[string], error) {
|
||||
rm.mu.Lock()
|
||||
defer rm.mu.Unlock()
|
||||
|
||||
e, ok := rm.entries[prefix]
|
||||
if !ok {
|
||||
logCallerF("No allowed IP reference found for prefix %v", prefix)
|
||||
return Ref[string]{}, nil
|
||||
}
|
||||
|
||||
if e.peers[peerKey] > 0 {
|
||||
logCallerF("Decreasing allowed IP ref count for prefix %v peer %s [peer %d -> %d, total %d -> %d, active %q]",
|
||||
prefix, peerKey, e.peers[peerKey], e.peers[peerKey]-1, e.total, e.total-1, e.active)
|
||||
e.peers[peerKey]--
|
||||
e.total--
|
||||
if e.peers[peerKey] == 0 {
|
||||
delete(e.peers, peerKey)
|
||||
}
|
||||
} else {
|
||||
logCallerF("No allowed IP reference found for prefix %v peer %s", prefix, peerKey)
|
||||
}
|
||||
|
||||
// If the peer currently installed in WireGuard still holds references, nothing to reprogram.
|
||||
// Keying the check on the active peer (not the one just released) makes this self-healing:
|
||||
// a prior swap whose remove/add failed leaves e.active pointing at a peer with no references,
|
||||
// and this retries the hand-off on the next Decrement instead of getting stuck.
|
||||
if e.active != "" && e.peers[e.active] > 0 {
|
||||
return Ref[string]{Count: e.total, Out: e.active}, nil
|
||||
}
|
||||
|
||||
// Detach the stale/gone active peer from WireGuard before reprogramming.
|
||||
if e.active != "" {
|
||||
if err := rm.remove(prefix, e.active); err != nil {
|
||||
return Ref[string]{Count: e.total, Out: e.active}, fmt.Errorf("remove allowed IP %v for peer %s: %w", prefix, e.active, err)
|
||||
}
|
||||
e.active = ""
|
||||
}
|
||||
|
||||
// Hand the prefix over to a surviving peer, or drop the entry when none remain.
|
||||
if survivor, ok := pickSurvivor(e.peers); ok {
|
||||
out, err := rm.add(prefix, survivor)
|
||||
if err != nil {
|
||||
return Ref[string]{Count: e.total, Out: ""}, fmt.Errorf("swap allowed IP %v to peer %s: %w", prefix, survivor, err)
|
||||
}
|
||||
e.active = out
|
||||
return Ref[string]{Count: e.total, Out: e.active}, nil
|
||||
}
|
||||
|
||||
delete(rm.entries, prefix)
|
||||
return Ref[string]{Count: 0, Out: ""}, nil
|
||||
}
|
||||
|
||||
// Flush removes all prefixes from WireGuard and clears the counter.
|
||||
func (rm *AllowedIPsRefCounter) Flush() error {
|
||||
rm.mu.Lock()
|
||||
defer rm.mu.Unlock()
|
||||
|
||||
var merr *multierror.Error
|
||||
for prefix, e := range rm.entries {
|
||||
if e.active == "" {
|
||||
continue
|
||||
}
|
||||
logCallerF("Flushing allowed IP for prefix %v peer %s", prefix, e.active)
|
||||
if err := rm.remove(prefix, e.active); err != nil {
|
||||
merr = multierror.Append(merr, fmt.Errorf("remove allowed IP %v for peer %s: %w", prefix, e.active, err))
|
||||
}
|
||||
}
|
||||
|
||||
clear(rm.entries)
|
||||
|
||||
return nberrors.FormatErrorOrNil(merr)
|
||||
}
|
||||
|
||||
// pickSurvivor deterministically selects a peer still referencing the prefix. WireGuard cannot do
|
||||
// multipath for a single prefix, so any surviving peer is a valid winner; the choice is made stable
|
||||
// (lowest peerKey) for predictable behavior and testability.
|
||||
func pickSurvivor(peers map[string]int) (string, bool) {
|
||||
if len(peers) == 0 {
|
||||
return "", false
|
||||
}
|
||||
keys := make([]string, 0, len(peers))
|
||||
for k := range peers {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys[0], true
|
||||
}
|
||||
241
client/internal/routemanager/refcounter/allowedips_test.go
Normal file
241
client/internal/routemanager/refcounter/allowedips_test.go
Normal file
@@ -0,0 +1,241 @@
|
||||
package refcounter
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/netip"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeWG models WireGuard's cryptokey routing: a prefix can be installed on exactly one peer.
|
||||
// failAdd/failRemove make the next add/remove fail once, to exercise the self-healing error paths.
|
||||
type fakeWG struct {
|
||||
installed map[netip.Prefix]string
|
||||
adds int
|
||||
removes int
|
||||
failAdd bool
|
||||
failRemove bool
|
||||
}
|
||||
|
||||
func newFakeWG() *fakeWG {
|
||||
return &fakeWG{installed: map[netip.Prefix]string{}}
|
||||
}
|
||||
|
||||
func (f *fakeWG) counter() *AllowedIPsRefCounter {
|
||||
return NewAllowedIPs(
|
||||
func(prefix netip.Prefix, peerKey string) (string, error) {
|
||||
if f.failAdd {
|
||||
f.failAdd = false
|
||||
return "", errors.New("add failed")
|
||||
}
|
||||
f.adds++
|
||||
f.installed[prefix] = peerKey
|
||||
return peerKey, nil
|
||||
},
|
||||
func(prefix netip.Prefix, peerKey string) error {
|
||||
if f.failRemove {
|
||||
f.failRemove = false
|
||||
return errors.New("remove failed")
|
||||
}
|
||||
f.removes++
|
||||
// only clear if this peer is the one installed, mirroring wg semantics
|
||||
if f.installed[prefix] == peerKey {
|
||||
delete(f.installed, prefix)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func mustPrefix(t *testing.T, s string) netip.Prefix {
|
||||
t.Helper()
|
||||
p, err := netip.ParsePrefix(s)
|
||||
if err != nil {
|
||||
t.Fatalf("parse prefix %q: %v", s, err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func mustIncrement(t *testing.T, c *AllowedIPsRefCounter, p netip.Prefix, peer string) Ref[string] {
|
||||
t.Helper()
|
||||
ref, err := c.Increment(p, peer)
|
||||
if err != nil {
|
||||
t.Fatalf("Increment(%v, %s): %v", p, peer, err)
|
||||
}
|
||||
return ref
|
||||
}
|
||||
|
||||
func mustDecrement(t *testing.T, c *AllowedIPsRefCounter, p netip.Prefix, peer string) Ref[string] {
|
||||
t.Helper()
|
||||
ref, err := c.Decrement(p, peer)
|
||||
if err != nil {
|
||||
t.Fatalf("Decrement(%v, %s): %v", p, peer, err)
|
||||
}
|
||||
return ref
|
||||
}
|
||||
|
||||
// TestAllowedIPs_SwapOnActivePeerRemoval reproduces the reported bug: two networks with the same
|
||||
// prefix routed by different peers. Removing the network whose peer is installed must hand the
|
||||
// prefix over to the surviving peer instead of leaving it on the removed one.
|
||||
func TestAllowedIPs_SwapOnActivePeerRemoval(t *testing.T) {
|
||||
f := newFakeWG()
|
||||
c := f.counter()
|
||||
p := mustPrefix(t, "10.44.8.0/24")
|
||||
|
||||
mustIncrement(t, c, p, "peerA")
|
||||
mustIncrement(t, c, p, "peerB")
|
||||
// First peer wins while both are present.
|
||||
if got := f.installed[p]; got != "peerA" {
|
||||
t.Fatalf("expected peerA installed, got %q", got)
|
||||
}
|
||||
|
||||
// Remove the active peer's network -> must swap to peerB.
|
||||
mustDecrement(t, c, p, "peerA")
|
||||
if got := f.installed[p]; got != "peerB" {
|
||||
t.Fatalf("BUG: prefix stuck on removed peer, want peerB got %q", got)
|
||||
}
|
||||
|
||||
// Remove the last one -> prefix gone.
|
||||
mustDecrement(t, c, p, "peerB")
|
||||
if _, ok := f.installed[p]; ok {
|
||||
t.Fatalf("expected prefix removed, still installed on %q", f.installed[p])
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowedIPs_RemoveNonActivePeer removing a non-installed peer must not touch WireGuard.
|
||||
func TestAllowedIPs_RemoveNonActivePeer(t *testing.T) {
|
||||
f := newFakeWG()
|
||||
c := f.counter()
|
||||
p := mustPrefix(t, "10.44.8.0/24")
|
||||
|
||||
mustIncrement(t, c, p, "peerA")
|
||||
mustIncrement(t, c, p, "peerB")
|
||||
removesBefore := f.removes
|
||||
|
||||
mustDecrement(t, c, p, "peerB")
|
||||
if f.installed[p] != "peerA" {
|
||||
t.Fatalf("active peer must stay peerA, got %q", f.installed[p])
|
||||
}
|
||||
if f.removes != removesBefore {
|
||||
t.Fatalf("removing a non-active peer must not call wg remove")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowedIPs_SamePeerMultipleRefs two references via the same peer must keep the prefix until
|
||||
// the last reference is released (the reason the per-peer count must be an int, not a set).
|
||||
func TestAllowedIPs_SamePeerMultipleRefs(t *testing.T) {
|
||||
f := newFakeWG()
|
||||
c := f.counter()
|
||||
p := mustPrefix(t, "10.44.8.0/24")
|
||||
|
||||
mustIncrement(t, c, p, "peerA")
|
||||
mustIncrement(t, c, p, "peerA")
|
||||
if f.adds != 1 {
|
||||
t.Fatalf("expected a single wg add for the same peer, got %d", f.adds)
|
||||
}
|
||||
|
||||
mustDecrement(t, c, p, "peerA")
|
||||
if f.installed[p] != "peerA" {
|
||||
t.Fatalf("prefix must stay while a reference remains, got %q", f.installed[p])
|
||||
}
|
||||
if f.removes != 0 {
|
||||
t.Fatalf("no wg remove expected while a reference remains, got %d", f.removes)
|
||||
}
|
||||
|
||||
mustDecrement(t, c, p, "peerA")
|
||||
if _, ok := f.installed[p]; ok {
|
||||
t.Fatalf("prefix must be removed after last reference")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowedIPs_RefCountAndActive checks the Ref returned to callers (used for the HA-disabled log).
|
||||
func TestAllowedIPs_RefCountAndActive(t *testing.T) {
|
||||
f := newFakeWG()
|
||||
c := f.counter()
|
||||
p := mustPrefix(t, "10.44.8.0/24")
|
||||
|
||||
ref := mustIncrement(t, c, p, "peerA")
|
||||
if ref.Count != 1 || ref.Out != "peerA" {
|
||||
t.Fatalf("want {1, peerA}, got {%d, %q}", ref.Count, ref.Out)
|
||||
}
|
||||
ref = mustIncrement(t, c, p, "peerB")
|
||||
if ref.Count != 2 || ref.Out != "peerA" {
|
||||
t.Fatalf("want {2, peerA}, got {%d, %q}", ref.Count, ref.Out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowedIPs_Flush removes everything installed and clears the counter.
|
||||
func TestAllowedIPs_Flush(t *testing.T) {
|
||||
f := newFakeWG()
|
||||
c := f.counter()
|
||||
p1 := mustPrefix(t, "10.44.8.0/24")
|
||||
p2 := mustPrefix(t, "10.44.9.0/24")
|
||||
|
||||
mustIncrement(t, c, p1, "peerA")
|
||||
mustIncrement(t, c, p2, "peerB")
|
||||
|
||||
if err := c.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.installed) != 0 {
|
||||
t.Fatalf("expected all prefixes removed, got %v", f.installed)
|
||||
}
|
||||
// After flush, a fresh increment must add again.
|
||||
mustIncrement(t, c, p1, "peerC")
|
||||
if f.installed[p1] != "peerC" {
|
||||
t.Fatalf("counter not reset after flush")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowedIPs_SelfHealAfterSwapAddError ensures a failed add during a swap does not permanently
|
||||
// strand the prefix: the next Decrement (or Increment) must retry and install a surviving peer.
|
||||
func TestAllowedIPs_SelfHealAfterSwapAddError(t *testing.T) {
|
||||
f := newFakeWG()
|
||||
c := f.counter()
|
||||
p := mustPrefix(t, "10.44.8.0/24")
|
||||
|
||||
mustIncrement(t, c, p, "peerA")
|
||||
mustIncrement(t, c, p, "peerB")
|
||||
mustIncrement(t, c, p, "peerC")
|
||||
|
||||
// Removing the active peerA triggers a swap to a survivor; make the add fail once.
|
||||
f.failAdd = true
|
||||
if _, err := c.Decrement(p, "peerA"); err == nil {
|
||||
t.Fatalf("expected error from failed swap add")
|
||||
}
|
||||
if _, ok := f.installed[p]; ok {
|
||||
t.Fatalf("nothing should be installed after a failed swap add, got %q", f.installed[p])
|
||||
}
|
||||
|
||||
// A later Decrement of a non-active survivor must retry the hand-off (self-heal), not stay stuck.
|
||||
ref := mustDecrement(t, c, p, "peerC")
|
||||
if got := f.installed[p]; got == "" {
|
||||
t.Fatalf("self-heal failed: prefix left unrouted after add recovered")
|
||||
}
|
||||
if ref.Out == "" {
|
||||
t.Fatalf("expected an active peer after self-heal, got empty")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowedIPs_SelfHealAfterRemoveError ensures a failed remove during a swap is retried instead
|
||||
// of leaving e.active stuck on a peer that no longer holds references.
|
||||
func TestAllowedIPs_SelfHealAfterRemoveError(t *testing.T) {
|
||||
f := newFakeWG()
|
||||
c := f.counter()
|
||||
p := mustPrefix(t, "10.44.8.0/24")
|
||||
|
||||
mustIncrement(t, c, p, "peerA")
|
||||
mustIncrement(t, c, p, "peerB")
|
||||
|
||||
// Releasing active peerA must detach it (remove) then add peerB; fail the remove once.
|
||||
f.failRemove = true
|
||||
if _, err := c.Decrement(p, "peerA"); err == nil {
|
||||
t.Fatalf("expected error from failed remove")
|
||||
}
|
||||
|
||||
// Next Decrement of the non-active survivor retries: removes stale peerA, installs peerB.
|
||||
mustDecrement(t, c, p, "peerB")
|
||||
// peerB had only one ref, so after retry the prefix is fully released.
|
||||
if _, ok := f.installed[p]; ok {
|
||||
t.Fatalf("expected prefix released after self-heal, still on %q", f.installed[p])
|
||||
}
|
||||
}
|
||||
@@ -5,5 +5,7 @@ import "net/netip"
|
||||
// RouteRefCounter is a Counter for Route, it doesn't take any input on Increment and doesn't use any output on Decrement
|
||||
type RouteRefCounter = Counter[netip.Prefix, struct{}, struct{}]
|
||||
|
||||
// AllowedIPsRefCounter is a Counter for AllowedIPs, it takes a peer key on Increment and passes it back to Decrement
|
||||
type AllowedIPsRefCounter = Counter[netip.Prefix, string, string]
|
||||
// AllowedIPsRefCounter tracks WireGuard AllowedIPs per prefix. Unlike the generic Counter it is peer-aware:
|
||||
// a prefix can be claimed by several peers at once and WireGuard allows a given prefix on exactly one peer,
|
||||
// so the counter records the per-peer reference count and swaps the installed peer when the active one is released.
|
||||
// See allowedips.go.
|
||||
|
||||
@@ -15,6 +15,11 @@ type Route struct {
|
||||
route *route.Route
|
||||
routeRefCounter *refcounter.RouteRefCounter
|
||||
allowedIPsRefcounter *refcounter.AllowedIPsRefCounter
|
||||
// currentPeerKey is the routing peer this watcher currently has the prefix installed on
|
||||
// (the HA winner elected by the watcher). It can differ from route.Peer and change on
|
||||
// failover, so it is recorded on AddAllowedIPs and used on RemoveAllowedIPs to decrement
|
||||
// the exact peer that was incremented.
|
||||
currentPeerKey string
|
||||
}
|
||||
|
||||
func NewRoute(params common.HandlerParams) *Route {
|
||||
@@ -52,12 +57,15 @@ func (r *Route) AddAllowedIPs(peerKey string) error {
|
||||
ref.Out,
|
||||
)
|
||||
}
|
||||
r.currentPeerKey = peerKey
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Route) RemoveAllowedIPs() error {
|
||||
if _, err := r.allowedIPsRefcounter.Decrement(r.route.Network); err != nil {
|
||||
return err
|
||||
var err error
|
||||
if _, decErr := r.allowedIPsRefcounter.Decrement(r.route.Network, r.currentPeerKey); decErr != nil {
|
||||
err = fmt.Errorf("remove allowed IP %s: %w", r.route.Network, decErr)
|
||||
}
|
||||
return nil
|
||||
r.currentPeerKey = ""
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -44,25 +44,10 @@ type Auth struct {
|
||||
// NewAuth instantiate Auth struct and validate the management URL
|
||||
func NewAuth(cfgPath string, mgmURL string) (*Auth, error) {
|
||||
inputCfg := profilemanager.ConfigInput{
|
||||
ConfigPath: cfgPath,
|
||||
ManagementURL: mgmURL,
|
||||
}
|
||||
|
||||
// Load the existing config when a config file is already present so an
|
||||
// interactive re-login reuses the peer's persisted WireGuard private key
|
||||
// (and thus its identity) instead of generating a fresh one. Generating a
|
||||
// new key registers a brand-new peer on the management server on every
|
||||
// re-auth (named after the fallback hostname). Only fall back to a fresh
|
||||
// in-memory config for the first-time login when no config file exists yet.
|
||||
// DirectUpdateOrCreateConfig uses non-atomic writes so it also works inside
|
||||
// the tvOS App Group sandbox where atomic temp-file+rename is blocked.
|
||||
var cfg *profilemanager.Config
|
||||
var err error
|
||||
if cfgPath != "" {
|
||||
cfg, err = profilemanager.DirectUpdateOrCreateConfig(inputCfg)
|
||||
} else {
|
||||
cfg, err = profilemanager.CreateInMemoryConfig(inputCfg)
|
||||
}
|
||||
cfg, err := profilemanager.CreateInMemoryConfig(inputCfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -1,27 +1,71 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs the NetBird daemon and brings the connection up in one container process.
|
||||
#
|
||||
# A thin wrapper is needed (rather than a one-line ENTRYPOINT) for two reasons:
|
||||
# 1. Two processes must run: the daemon (`service run`, long-lived) and a
|
||||
# one-shot `up` that brings the connection up.
|
||||
# 2. Signal handling: as PID 1 the wrapper must forward SIGTERM/SIGINT to the
|
||||
# daemon so it tears down WireGuard and deregisters ephemeral peers on
|
||||
# `docker stop`. Without this the daemon would be killed uncleanly.
|
||||
#
|
||||
# `netbird up` waits for the daemon to become ready on its own, so no readiness
|
||||
# poll is needed here.
|
||||
set -eEuo pipefail
|
||||
|
||||
: ${NB_ENTRYPOINT_SERVICE_TIMEOUT:="30"}
|
||||
NETBIRD_BIN="${NETBIRD_BIN:-"netbird"}"
|
||||
export NB_LOG_FILE="${NB_LOG_FILE:-"console,/var/log/netbird/client.log"}"
|
||||
service_pids=()
|
||||
|
||||
daemon=""
|
||||
cleanup() { [[ -n "${daemon}" ]] && kill -TERM "${daemon}" 2>/dev/null || true; }
|
||||
trap cleanup SIGTERM SIGINT EXIT
|
||||
_log() {
|
||||
# mimic Go logger's output for easier parsing
|
||||
# 2025-04-15T21:32:00+08:00 INFO client/internal/config.go:495: setting notifications to disabled by default
|
||||
printf "$(date -Isec) ${1} ${BASH_SOURCE[1]}:${BASH_LINENO[1]}: ${2}\n" "${@:3}" >&2
|
||||
}
|
||||
|
||||
"${NETBIRD_BIN}" service run &
|
||||
daemon=$!
|
||||
info() {
|
||||
_log INFO "$@"
|
||||
}
|
||||
|
||||
"${NETBIRD_BIN}" up
|
||||
warn() {
|
||||
_log WARN "$@"
|
||||
}
|
||||
|
||||
wait "${daemon}"
|
||||
on_exit() {
|
||||
info "Shutting down NetBird daemon..."
|
||||
if test "${#service_pids[@]}" -gt 0; then
|
||||
info "terminating service process IDs: ${service_pids[@]@Q}"
|
||||
kill -TERM "${service_pids[@]}" 2>/dev/null || true
|
||||
wait "${service_pids[@]}" 2>/dev/null || true
|
||||
else
|
||||
info "there are no service processes to terminate"
|
||||
fi
|
||||
}
|
||||
|
||||
wait_for_daemon_startup() {
|
||||
local timeout="${1}"
|
||||
if [[ "${timeout}" -eq 0 ]]; then
|
||||
info "not waiting for daemon startup due to zero timeout."
|
||||
return
|
||||
fi
|
||||
|
||||
local deadline=$((SECONDS + timeout))
|
||||
while [[ "${SECONDS}" -lt "${deadline}" ]]; do
|
||||
if "${NETBIRD_BIN}" status --check live 2>/dev/null; then
|
||||
return
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
warn "daemon did not become responsive after ${timeout} seconds, exiting..."
|
||||
exit 1
|
||||
}
|
||||
|
||||
connect() {
|
||||
info "running 'netbird up'..."
|
||||
"${NETBIRD_BIN}" up
|
||||
return $?
|
||||
}
|
||||
|
||||
main() {
|
||||
trap 'on_exit' SIGTERM SIGINT EXIT
|
||||
"${NETBIRD_BIN}" service run &
|
||||
service_pids+=("$!")
|
||||
info "registered new service process 'netbird service run', currently running: ${service_pids[@]@Q}"
|
||||
|
||||
wait_for_daemon_startup "${NB_ENTRYPOINT_SERVICE_TIMEOUT}"
|
||||
connect
|
||||
|
||||
wait "${service_pids[@]}"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
@@ -995,13 +995,8 @@ type StatusResponse struct {
|
||||
// Unset when the peer is not SSO-registered or login expiration is disabled.
|
||||
// The UI derives "warning active" from this value and its own clock.
|
||||
SessionExpiresAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=sessionExpiresAt,proto3" json:"sessionExpiresAt,omitempty"`
|
||||
// daemonReady reports whether the daemon has finished starting up and is
|
||||
// serving RPCs backed by a running engine. Older daemons never set this
|
||||
// (it defaults to false); clients must treat an unset value as "unknown"
|
||||
// and fall back to their previous readiness heuristics for compatibility.
|
||||
DaemonReady bool `protobuf:"varint,5,opt,name=daemonReady,proto3" json:"daemonReady,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *StatusResponse) Reset() {
|
||||
@@ -1062,13 +1057,6 @@ func (x *StatusResponse) GetSessionExpiresAt() *timestamppb.Timestamp {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *StatusResponse) GetDaemonReady() bool {
|
||||
if x != nil {
|
||||
return x.DaemonReady
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type DownRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
@@ -7095,15 +7083,14 @@ const file_daemon_proto_rawDesc = "" +
|
||||
"\x11getFullPeerStatus\x18\x01 \x01(\bR\x11getFullPeerStatus\x12(\n" +
|
||||
"\x0fshouldRunProbes\x18\x02 \x01(\bR\x0fshouldRunProbes\x12'\n" +
|
||||
"\fwaitForReady\x18\x03 \x01(\bH\x00R\fwaitForReady\x88\x01\x01B\x0f\n" +
|
||||
"\r_waitForReady\"\xec\x01\n" +
|
||||
"\r_waitForReady\"\xca\x01\n" +
|
||||
"\x0eStatusResponse\x12\x16\n" +
|
||||
"\x06status\x18\x01 \x01(\tR\x06status\x122\n" +
|
||||
"\n" +
|
||||
"fullStatus\x18\x02 \x01(\v2\x12.daemon.FullStatusR\n" +
|
||||
"fullStatus\x12$\n" +
|
||||
"\rdaemonVersion\x18\x03 \x01(\tR\rdaemonVersion\x12F\n" +
|
||||
"\x10sessionExpiresAt\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\x10sessionExpiresAt\x12 \n" +
|
||||
"\vdaemonReady\x18\x05 \x01(\bR\vdaemonReady\"\r\n" +
|
||||
"\x10sessionExpiresAt\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\x10sessionExpiresAt\"\r\n" +
|
||||
"\vDownRequest\"\x0e\n" +
|
||||
"\fDownResponse\"P\n" +
|
||||
"\x10GetConfigRequest\x12 \n" +
|
||||
|
||||
@@ -291,11 +291,6 @@ message StatusResponse{
|
||||
// Unset when the peer is not SSO-registered or login expiration is disabled.
|
||||
// The UI derives "warning active" from this value and its own clock.
|
||||
google.protobuf.Timestamp sessionExpiresAt = 4;
|
||||
// daemonReady reports whether the daemon has finished starting up and is
|
||||
// serving RPCs backed by a running engine. Older daemons never set this
|
||||
// (it defaults to false); clients must treat an unset value as "unknown"
|
||||
// and fall back to their previous readiness heuristics for compatibility.
|
||||
bool daemonReady = 5;
|
||||
}
|
||||
|
||||
message DownRequest {}
|
||||
|
||||
@@ -104,12 +104,6 @@ type Server struct {
|
||||
persistSyncResponse bool
|
||||
isSessionActive atomic.Bool
|
||||
|
||||
// ready is set once the daemon has finished startup and is serving RPCs
|
||||
// backed by a running engine (see SetReady, called after Start succeeds and
|
||||
// the service is registered). Reported via StatusResponse.DaemonReady so
|
||||
// clients can wait deterministically instead of polling heuristically.
|
||||
ready atomic.Bool
|
||||
|
||||
cpuProfileBuf *bytes.Buffer
|
||||
cpuProfiling bool
|
||||
|
||||
@@ -164,14 +158,6 @@ func New(ctx context.Context, logFile string, configFile string, profilesDisable
|
||||
return s
|
||||
}
|
||||
|
||||
// SetReady marks the daemon as fully started and serving RPCs. It is called by
|
||||
// the service controller once Start has succeeded and the DaemonService is
|
||||
// registered, so a subsequent Status RPC reports DaemonReady=true. Safe for
|
||||
// concurrent use.
|
||||
func (s *Server) SetReady() {
|
||||
s.ready.Store(true)
|
||||
}
|
||||
|
||||
func (s *Server) Start() error {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
@@ -1436,7 +1422,7 @@ func (s *Server) buildStatusResponse(ctx context.Context, msg *proto.StatusReque
|
||||
s.isSessionActive.Store(false)
|
||||
}
|
||||
|
||||
statusResponse := proto.StatusResponse{Status: string(status), DaemonVersion: version.NetbirdVersion(), DaemonReady: s.ready.Load()}
|
||||
statusResponse := proto.StatusResponse{Status: string(status), DaemonVersion: version.NetbirdVersion()}
|
||||
|
||||
if deadline := s.statusRecorder.GetSessionExpiresAt(); !deadline.IsZero() {
|
||||
statusResponse.SessionExpiresAt = timestamppb.New(deadline)
|
||||
|
||||
@@ -17,7 +17,8 @@ RUN apk add --no-cache bash ca-certificates ip6tables iproute2 iptables
|
||||
ENV NETBIRD_BIN="/usr/local/bin/netbird" \
|
||||
NB_LOG_FILE="console,/var/log/netbird/client.log" \
|
||||
NB_DAEMON_ADDR="unix:///var/run/netbird.sock" \
|
||||
NB_ENABLE_CAPTURE="false"
|
||||
NB_ENABLE_CAPTURE="false" \
|
||||
NB_ENTRYPOINT_SERVICE_TIMEOUT="30"
|
||||
ENTRYPOINT [ "/usr/local/bin/netbird-entrypoint.sh" ]
|
||||
COPY client/netbird-entrypoint.sh /usr/local/bin/netbird-entrypoint.sh
|
||||
COPY --from=builder /out/netbird /usr/local/bin/netbird
|
||||
|
||||
Reference in New Issue
Block a user